ESPnet2で始めるEnd-to-End音声処理

$ git clone https://github.com/kaldi-asr/kaldi.git
$ git clone https://github.com/espnet/espnet.git -b v.0.9.4

$ cd espnet/tools

$ ln -s $(pwd)/../../kaldi .

$ . ./setup_cuda_env.sh /usr/local/cuda

$ ./setup_anaconda.sh venv

$ make

$ make pyopenjtalk.done

$ . ./activate_python.sh

from espnet_model_zoo.downloader import ModelDownloader
from espnet2.bin.tts_inference import Text2Speech

# Create E2E-TTS model instance
d = ModelDownloader()
text2speech = Speech2Text(
    # Specify the tag
    d.download_and_unpack("kan-bayashi/jsut_fastspeech2")
)

# Synthesis with a given text
wav, feats, feats_denorm, *_ = text2speech(
	"あらゆる現実を、全て自分の方へねじ曲げたのだ。"
)

import soundfile as sf
from espnet_model_zoo.downloader import ModelDownloader
from espnet2.bin.asr_inference import Speech2Text

# Create E2E-ASR model instance
d = ModelDownloader()
speech2text = Speech2Text(
    # Specify task and corpus
    **d.download_and_unpack(task="asr", corpus="librispeech")
)

# Recognition with a given audio
wav, fs = sf.read("/path/to/sample.wav")
text, token, *_ = speech2text(wav)[0]

# ASR recipe                       # TTS recipe
egs2/jsut/asr1/                    egs2/jsut/tts1/
 - conf/                            - conf/
 - scripts/                         - scripts/
 - pyscripts/                       - pyscripts/
 - steps/                           - steps/
 - utils/                           - utils/
 - local/                           - local/
 - db.sh                            - db.sh
 - path.sh                          - path.sh
 - cmd.sh                           - cmd.sh
 - run.sh                           - run.sh
 - asr.sh                           - tts.sh

$ cd espnet/egs2/TEMPLATE/asr1
$ ./asr.sh --help
2020-09-14T15:38:49(asr.sh:208:main) ./asr.sh --help
Usage: ./asr.sh --train-set <train_set_name> --valid-set <valid_set_name> --test_sets <test_set_names> --srctexts <srctexts>

Options:
    # General configuration
    --stage          # Processes starts from the specified stage(default="1").
    --stop_stage     # Processes is stopped at the specified stage(default="10000").
    --skip_data_prep # Skip data preparation stages(default="false").
    --skip_train     # Skip training stages(default="false").
    --skip_eval      # Skip decoding and evaluation stages(default="false").
    --skip_upload    # Skip packing and uploading stages(default="true").
    --ngpu           # The number of gpus("0" uses cpu, otherwise use gpu, default="1").
    --num_nodes      # The number of nodes(default="1").
    --nj             # The number of parallel jobs(default="32").
    --inference_nj   # The number of parallel jobs in decoding(default="32").
    --gpu_inference  # Whether to perform gpu decoding(default="false").
    --dumpdir        # Directory to dump features(default="dump").
    --expdir         # Directory to save experiments(default="exp").
    --python         # Specify python to execute espnet commands(default="python3").

    # Data preparation related
    --local_data_opts # The options given to local/data.sh(default="").

    # Speed perturbation related
    --speed_perturb_factors # speed perturbation factors, e.g. "0.9 1.0 1.1"(separated by space, default="").

    # Feature extraction related
    --feats_type       # Feature type(raw, fbank_pitch or extracted, default="raw").
    --audio_format     # Audio format(only in feats_type=raw, default="flac").
    --fs               # Sampling rate(default="16k").
    --min_wav_duration # Minimum duration in second(default="0.1").
    --max_wav_duration # Maximum duration in second(default="20").

    # Tokenization related
    --token_type              # Tokenization type(char or bpe, default="bpe").
    --nbpe                    # The number of BPE vocabulary(default="30").
    --bpemode                 # Mode of BPE(unigram or bpe, default="unigram").
    --oov                     # Out of vocabulary symbol(default="<unk>").
    --blank                   # CTC blank symbol(default="<blank>").
    --sos_eos                 # sos and eos symbole(default="<sos/eos>").
    --bpe_input_sentence_size # Size of input sentence for BPE(default="100000000").
    --bpe_nlsyms              # Non-linguistic symbol list for sentencepiece, separated by a comma.(default="").
    --bpe_char_cover          # Character coverage when modeling BPE(default="1.0").

    # Language model related
    --lm_tag          # Suffix to the result dir for language model training(default="").
    --lm_exp          # Specify the direcotry path for LM experiment.
                      # If this option is specified, lm_tag is ignored(default="").
    --lm_config       # Config for language model training(default="").
    --lm_args         # Arguments for language model training(default="").
                      # e.g., --lm_args "--max_epoch 10"
                      # Note that it will overwrite args in lm config.
    --use_word_lm     # Whether to use word language model(default="false").
    --word_vocab_size # Size of word vocabulary(default="10000").
    --num_splits_lm   # Number of splitting for lm corpus(default="1").

    # ASR model related
    --asr_tag          # Suffix to the result dir for asr model training(default="").
    --asr_exp          # Specify the direcotry path for ASR experiment.
                       # If this option is specified, asr_tag is ignored(default="").
    --asr_config       # Config for asr model training(default="").
    --asr_args         # Arguments for asr model training(default="").
                       # e.g., --asr_args "--max_epoch 10"
                       # Note that it will overwrite args in asr config.
    --feats_normalize  # Normalizaton layer type(default="global_mvn").
    --num_splits_asr   # Number of splitting for lm corpus (default="1").

    # Decoding related
    --inference_tag       # Suffix to the result dir for decoding(default="").
    --inference_config    # Config for decoding(default="").
    --inference_args      # Arguments for decoding(default="").
                          # e.g., --inference_args "--lm_weight 0.1"
                          # Note that it will overwrite args in inference config.
    --inference_lm        # Language modle path for decoding(default="valid.loss.ave.pth").
    --inference_asr_model # ASR model path for decoding(default="valid.acc.ave.pth").
    --download_model      # Download a model from Model Zoo and use it for decoding(default="").

    # [Task dependent] Set the datadir name created by local/data.sh
    --train_set     # Name of training set(required).
    --valid_set     # Name of validation set used for monitoring/tuning network training(required).
    --test_sets     # Names of test sets.
                    # Multiple items(e.g., both dev and eval sets) can be specified(required).
    --srctexts      # Used for the training of BPE and LM and the creation of a vocabulary list(required).
    --lm_dev_text   # Text file path of language model development set(default="").
    --lm_test_text  # Text file path of language model evaluation set(default="").
    --nlsyms_txt    # Non-linguistic symbol list if existing(default="none").
    --cleaner       # Text cleaner(default="none").
    --g2p           # g2p method(default="none").
    --lang          # The language type of corpus(default=noinfo).
    --asr_speech_fold_length # fold_length for speech data during ASR training(default="800").
    --asr_text_fold_length   # fold_length for text data during ASR training(default="150").
    --lm_fold_length         # fold_length for LM training(default="150").

$ cd espnet/egs2/jsut/asr1

$ vim cmd.sh
$ vim db.sh

$ ./run.sh

#########################
#    Stage 1-5で作成     #
#########################
# ダウンロードされたコーパス
- downloads/
  ├ jsut-lab/
  └ jsut_ver1.1/
# データディレクトリ
- data/
  ├ dev/        # 検証セット
  ├ eval1/      # 評価セット
  ├ token_list/ # トークンリスト(辞書)
  └ tr_no_dev/  # 学習セット
# 特徴量ディレクトリ
- dump/
  └ raw/
    ├ dev/       # 検証セット
    ├ eval1/     # 評価セット
    ├ tr_no_dev/ # 学習セット
    └ srctexts   # 言語モデル学習用テキスト
                  #(辞書作成用兼)
#########################
#    Stage 6以降で作成    #
#########################
# 実験ディレクトリ
- exp/
  ├ asr_stats_raw/
  │ ├ train/ # 学習セットASR統計量
  │ └ valid/ # 検証セットASR統計量
  ├ asr_train_asr_rnn_raw_char/
  │ ├ att_ws/        # Attentionプロット
  │ ├ decode_*/      # デコーディング結果
  │ ├ tensorboard/   # Tensorboardログ
  │ ├ images/        # 学習曲線プロット
  │ ├ README.md      # 評価結果のサマリー
  │ ├ train.log      # 学習ログ
  │ ├ *.pth          # モデルパラメータ
  │ └ checkpoint.pth # モデルパラメータ(Optimizer等を含む)
  ├ lm_stats/
  │ ├ train/ # 学習セットLM統計量
  │ └ valid/ # 検証セットLM統計量
  └ lm_train_lm_char/
    ├ perplexity_*/  # PPL評価結果
    ├ tensorboard/   # Tensorboardログ
    ├ images/        # 学習曲線プロット
    ├ train.log      # 学習ログ
    ├ *.pth          # モデルパラメータ
    └ checkpoint.pth # モデルパラメータ(Optimizer等を含む)

# stage 1のみを実行
$ ./run.sh --stage 1 --stop-stage 1
# stage 2のみを実行
$ ./run.sh --stage 2 --stop-stage 2

$ ./run.sh --ngpu 3

$ CUDA_VISIBLE_DEVICES=0,1,2 ./run.sh --ngpu 3

$ ./run.sh --ngpu 4 --num_nodes 4

$ ./run.sh --asr_config conf/train_asr_transformer.yaml

$ ./run.sh --stage 10 --asr_config conf/train_asr_transformer.yaml

$ ./run.sh --stage 10 \
  --asr_config conf/train_asr_rnn.yaml \
  --asr_args "--batch_size 64"

$ ./run.sh --stage 10 \
  --asr_config conf/train_asr_rnn.yaml \
  --asr_args "--batch_size 64 --optim_conf lr=0.1"

$ ./run.sh --stage 10 \
  --asr_config conf/train_asr_rnn.yaml \
  --asr_args "--batch_size 64 --optim_conf lr=0.1" \
  --asr_tag "train_rnn_batchs_size_64_lr_0.1"

# Python環境のアクティベート
$ . ./path.sh

# Helpメッセージの表示
$ python3 -m espnet2.bin.lm_train --help
$ python3 -m espnet2.bin.asr_train --help
$ python3 -m espnet2.bin.lm_inference --help
$ python3 -m espnet2.bin.asr_inference --help

# デフォルトコンフィグを表示
$ python3 -m espnet2.bin.lm_train --print_config
$ python3 -m espnet2.bin.asr_train --print_config

# 与えられたコンフィグをファイルを反映したコンフィグを表示
$ python3 -m espnet2.bin.lm_train --print_config --config conf/train_lm.yaml
$ python3 -m espnet2.bin.asr_train --print_config --config conf/train_asr_transformer.yaml

$ cd espnet/egs2/TEMPLATE/tts1
$ ./tts.sh --help
2020-09-14T15:59:21(tts.sh:187:main) ./tts.sh --help
Usage: ./tts.sh --train-set "<train_set_name>" --valid-set "<valid_set_name>" --test_sets "<test_set_names>" --srctexts "<srctexts>"

Options:
    # General configuration
    --stage          # Processes starts from the specified stage(default="1").
    --stop_stage     # Processes is stopped at the specified stage(default="10000").
    --skip_data_prep # Skip data preparation stages(default="false").
    --skip_train     # Skip training stages(default="false").
    --skip_eval      # Skip decoding and evaluation stages(default="false").
    --skip_upload    # Skip packing and uploading stages(default="true").
    --ngpu           # The number of gpus("0" uses cpu, otherwise use gpu, default="1").
    --num_nodes      # The number of nodes(default="1").
    --nj             # The number of parallel jobs(default="32").
    --inference_nj   # The number of parallel jobs in decoding(default="32").
    --gpu_inference  # Whether to perform gpu decoding(default="false").
    --dumpdir        # Directory to dump features(default="dump").
    --expdir         # Directory to save experiments(default="exp").
    --python         # Specify python to execute espnet commands(default="python3").

    # Data prep related
    --local_data_opts # Options to be passed to local/data.sh(default="").

    # Feature extraction related
    --feats_type       # Feature type(fbank or stft or raw, default="raw").
    --audio_format     # Audio format(only in feats_type=raw, default="flac").
    --min_wav_duration # Minimum duration in second(default="0.1").
    --max_wav_duration # Maximum duration in second(default="20").
    --fs               # Sampling rate(default="16000").
    --fmax             # Maximum frequency of Mel basis(default="7600").
    --fmin             # Minimum frequency of Mel basis(default="80").
    --n_mels           # The number of mel basis(default="80").
    --n_fft            # The number of fft points(default="1024").
    --n_shift          # The number of shift points(default="256").
    --win_length       # Window length(default="null").
    --f0min            # Maximum f0 for pitch extraction(default="80").
    --f0max            # Minimum f0 for pitch extraction(default="400").
    --oov              # Out of vocabrary symbol(default="<unk>").
    --blank            # CTC blank symbol(default="<blank>").
    --sos_eos          # sos and eos symbole(default="<sos/eos>").

    # Training related
    --train_config  # Config for training(default="").
    --train_args    # Arguments for training(default="").
                    # e.g., --train_args "--max_epoch 1"
                    # Note that it will overwrite args in train config.
    --tag           # Suffix for training directory(default="").
    --tts_exp       # Specify the direcotry path for experiment.
                    # If this option is specified, tag is ignored(default="").
    --tts_stats_dir # Specify the direcotry path for statistics.
                    # If empty, automatically decided(default="").
    --num_splits    # Number of splitting for tts corpus(default="1").
    --write_collected_feats # Whether to dump features in statistics collection(default="false").

    # Decoding related
    --inference_config  # Config for decoding(default="").
    --inference_args    # Arguments for decoding,(default="").
                        # e.g., --inference_args "--threshold 0.75"
                        # Note that it will overwrite args in inference config.
    --inference_tag     # Suffix for decoding directory(default="").
    --inference_model   # Model path for decoding(default=train.loss.ave.pth).
    --griffin_lim_iters # The number of iterations of Griffin-Lim(default=4).
    --download_model    # Download a model from Model Zoo and use it for decoding(default="").

    # [Task dependent] Set the datadir name created by local/data.sh.
    --train_set          # Name of training set(required).
    --valid_set          # Name of validation set used for monitoring/tuning network training(required).
    --test_sets          # Names of test sets(required).
                         # Note that multiple items(e.g., both dev and eval sets) can be specified.
    --srctexts           # Texts to create token list(required).
                         # Note that multiple items can be specified.
    --nlsyms_txt         # Non-linguistic symbol list(default="none").
    --token_type         # Transcription type(default="phn").
    --cleaner            # Text cleaner(default="tacotron").
    --g2p                # g2p method(default="g2p_en").
    --lang               # The language type of corpus(default="noinfo").
    --text_fold_length   # Fold length for text data(default="150").
    --speech_fold_length # Fold length for speech data(default="800").

$ cd espnet/egs2/jsut/tts1
$ ./run.sh

# Stage 1のみを実行
$ ./run.sh --stage 1 --stop-stage 1
# Stage 2のみを実行
$ ./run.sh --stage 2 --stop-stage 2

#########################
#    Stage 1-4で作成    #
#########################
# ダウンロードされたコーパス
- downloads/
  ├ jsut-lab/
  └ jsut_ver1.1/
# データディレクトリ
- data/
  ├ dev/        # 検証セット
  ├ eval1/      # 評価セット
  ├ token_list/ # トークンリスト(辞書)
  └ tr_no_dev/  # 学習セット
# 特徴量ディレクトリ
- dump/
  └ raw/
    ├ dev/       # 検証セット
    ├ eval1/     # 評価セット
    ├ tr_no_dev/ # 学習セット
    └ srctexts   # 辞書作成用テキスト
#########################
#   Stage 5以降で作成   #
#########################
# 実験ディレクトリ
- exp/
  ├ tts_stats_*/
  │ ├ train/ # 学習セット統計量
  │ └ valid/ # 検証セット統計量
  └ tts_train_*/
    ├ att_ws/        # Attentionプロット
    ├ decode_*/      # デコーディング結果
    ├ tensorboard/   # Tensorboardログ
    ├ images/        # 学習曲線プロット
    ├ train.log      # 学習ログ
    ├ *.pth          # モデルパラメータ
    └ checkpoint.pth # モデルパラメータ(Optimizer等を含む)

- exp/tts_train_*/decode_*/
 ├ dev/   # 検証セットのデコード結果
 └ eval1/ # 評価セットのデコード結果
   ├ att_ws/      # Attentionプロット
   ├ probs/       # 生成打切確率プロット
   ├ denorm/      # 逆正規化後生成特徴量
   ├ norm/        # 正規化後生成特徴量
   ├ wav/         # 生成音声
   ├ durations    # 各音素の継続長
   ├ feats_type   # 特徴量の種類
   ├ focus_rates  # フォーカスレート
   └ speech_shape # 生成特徴量のシェイプ情報

# Python環境のアクティベート
$ . ./path.sh

# Helpメッセージの表示
$ python3 -m espnet2.bin.tts_train --help
$ python3 -m espnet2.bin.tts_inference --help

# デフォルトコンフィグを表示
$ python3 -m espnet2.bin.tts_train --print_config

# 与えられたコンフィグをファイルを反映したコンフィグを表示
$ python3 -m espnet2.bin.tts_train --print_config --config conf/train.yaml

$ ./run.sh --stage 7 \
  --tts_exp exp/<teacher_model_dir> \
  --test_sets "tr_no_dev dev eval1"

$ ./run.sh --stage 6 \
  --teacher_dumpdir exp/<teacher_model_dir>/decode_train.loss.ave \
  --train_config conf/tuning/train_fastspeech.yaml

$ ./run.sh --stage 7 \
  --tts_exp exp/<teacher_model_dir> \
  --test_sets "tr_no_dev dev eval1" \
  --inference_args "--use_teacher_forcing true"

$ ./run.sh --stage 5 \
  --write_collected_feats true \
  --teacher_dumpdir exp/<teacher_model_dir>/decode_use_teacher_forcingtrue_train.loss.ave \
  --tts_stats_dir exp/<teacher_model_dir>/decode_use_teacher_forcingtrue_train.loss.ave/stats \
  --train_config conf/tuning/train_fastspeech2.yaml