DASB TTS Implementation

README.md

@@ -21,7 +21,7 @@ The SpeechBrain Benchmarks currently include the following:
 
 - [MOABB](https://github.com/speechbrain/benchmarks/tree/main/benchmarks/MOABB) - A benchmark designed for evaluating neural models in well-known EEG tasks like motor imagery, P300, and SSVEP.
 
-- [DASB](https://github.com/speechbrain/benchmarks/tree/main/benchmarks/DASB) - A benchmark designed for evaluating discrete audio tokens across a wide range of discriminative
+- [DASB](https://github.com/speechbrain/benchmarks/tree/DASB/benchmarks/DASB) - A benchmark designed for evaluating discrete audio tokens across a wide range of discriminative
 and  generative tasks.
 
 

benchmarks/DASB/LJSpeech/TTS/tokotron/evaluate.py

@@ -51,17 +51,7 @@ def __init__(self, hparams, create_waveform_fn, device):
         else:
             self.evaluators = {}
 
-        bulk_evaluators = getattr(self.hparams, "bulk_evaluators", {})
-        if bulk_evaluators:
-            self.bulk_evaluators = {
-                key: evaluator_f()
-                for key, evaluator_f in bulk_evaluators.items()
-                if key in self.enabled_evaluators
-            }
-        else:
-            self.bulk_evaluators = {}
-
-        if not self.evaluators and not self.bulk_evaluators:
+        if not self.evaluators:
             logger.warn(
                 "No evaluators were defined - this run will produce samples only"
             )
@@ -98,9 +88,7 @@ def on_evaluate_start(self, stage, epoch):
         self.create_reports()
         self.modules.model.show_inference_progress = False
         self.item_ids = []
-        details_keys = list(self.evaluators.keys()) + list(
-            self.bulk_evaluators.keys()
-        )
+        details_keys = list(self.evaluators.keys())
         self.details = {evaluator_key: [] for evaluator_key in details_keys}
         self.sample_text = []
         self.sample_file_names = []
@@ -141,7 +129,6 @@ def on_evaluate_end(self):
         dataset : speechbrain.dataio.dataset.DynamicItemDataset
             a dataset
         """
-        self.evaluate_bulk()
         self.write_summary()
         logger.info("Evaluation done")
 
@@ -182,19 +169,6 @@ def get_report_columns(self, evaluator_key):
                 wavs_ref=bogus_wavs,
                 length_ref=bogus_length,
             )
-        else:
-            bogus_file_name = self.output_folder / "bogus.wav"
-            evaluator = self.bulk_evaluators[evaluator_key]
-            sb.dataio.dataio.write_audio(
-                str(bogus_file_name),
-                bogus_wavs[0].cpu(),
-                samplerate=self.hparams.model_sample_rate,
-            )
-            result = evaluator.evaluate_files(
-                file_names=[bogus_file_name],
-                text=["BOGUS"],
-                file_names_ref=[bogus_file_name],
-            )
 
         return ["uttid"] + list(result.details.keys())
 
@@ -228,19 +202,6 @@ def evaluate_batch(self, batch):
                 self.write_result(evaluator_key, batch.uttid, details)
                 self.details[evaluator_key].extend(details)
 
-    def evaluate_bulk(self):
-        """Runs all configured bulk evaluators, which evaluate a directory
-        of files - rather than one file at a time"""
-        for evaluator_key, evaluator in self.bulk_evaluators.items():
-            result = evaluator.evaluate_files(
-                file_names=self.sample_file_names,
-                text=self.sample_text,
-                file_names_ref=self.ref_file_names,
-            )
-            self.details[evaluator_key].append(result.details)
-            details = undo_batch(result.details)
-            self.write_result(evaluator_key, self.item_ids, details)
-
     def write_result(self, evaluator_key, uttid, details):
         """Outputs the result details to the report for the specified evaluator
 

benchmarks/DASB/LJSpeech/TTS/tokotron/hparams/eval.yaml

@@ -1,50 +1,56 @@
+# ############################################################################
+# Evaluation Hyperparameters
+# Common to old models, appended to main hyperparameters
+#
+# Authors:  Artem Ploujnikov
+# ############################################################################
+
+eval_enabled: True
 eval_sample_rate: 16000
 eval_samples: null
 eval_interval: 1
 eval_asr_type: whisper
-eval_asr_source: !apply:speechbrain.utils.hparams.choice
-  value: !ref <eval_asr_type>
-  choices:
-    encoder_decoder: speechbrain/asr-transformer-transformerlm-librispeech
-    whisper: openai/whisper-small
+eval_asr_source: openai/whisper-small
 evaluations: utmos,asr
 tmp_folder: null
-utmos_batch_size: 8
-utmos_model_path: ./utmos
-utmos_ckpt_name: epoch=3-step=7459.ckpt
-utmos_ckpt_path: !ref <utmos_model_path>/<utmos_ckpt_name>
-utmos_use_python: True
-utmos_script: predict.py
-
-
-eval_asr: !apply:speechbrain.utils.hparams.choice
-  value: !ref <eval_asr_type>
-  choices:
-    encoder_decoder: !name:eval.EncoderDecoderASRSpeechEvaluator
-      source: !ref <eval_asr_source>
-      sample_rate: !ref <eval_sample_rate>
-      overrides:
-        lm_weight: 0.0
-    whisper: !name:eval.WhisperASRSpeechEvaluator
-      source: !ref <eval_asr_source>
-      sample_rate: !ref <eval_sample_rate>
-      savedir: !ref <pretrained_model_save_folder>
+eval_utmos_source: chaanks/wav2vec2-small
+eval_utmos_save_path: !ref <pretrained_model_save_folder>/utmos
+eval_utmos_model_name: utmos.ckpt
+eval_utmos_model_url: https://huggingface.co/chaanks/UTMOS/resolve/main
+eval_utmos_domain_id: null
+eval_utmos_judge_id: null
+eval_perf: False
+
+
+eval_utmos: !name:eval.UTMOSSpeechEvaluator
+  source: !ref <eval_utmos_source>
+  save_path: !ref <eval_utmos_save_path>
+  model_name: !ref <eval_utmos_model_name>
+  model_url: !ref <eval_utmos_model_url>
+  domain_id: !ref <eval_utmos_domain_id>
+  judge_id: !ref <eval_utmos_judge_id>
+
+eval_asr: !name:eval.WhisperASRSpeechEvaluator
+  source: !ref <eval_asr_source>
+  sample_rate: !ref <eval_sample_rate>
+  savedir: !ref <pretrained_model_save_folder>
 
 evaluators:
+  utmos: !ref <eval_utmos>
   asr: !ref <eval_asr>
 
-bulk_evaluators:
-  utmos: !name:eval.UTMOSSpeechEvaluator
-    model_path: !ref <utmos_model_path>
-    output_folder: !ref <output_folder>
-    ckpt_path: !ref <utmos_ckpt_path>
-    batch_size: !ref <utmos_batch_size>
-    script: !ref <utmos_script>
-    use_python: !ref <utmos_use_python>
-    tmp_folder: !ref <tmp_folder>
-
 eval_summary:
   asr:
     descriptive: ["wer", "cer", "wer_ref", "cer_ref", "dwer", "dcer"]
   utmos:
     descriptive: ["utmos"]
+
+eval_summary_log:
+  utmos: utmos_utmos_mean
+  dwer: asr_dwer_median
+
+eval_threshold:
+  dwer_max: 90.0
+
+eval_threshold_set:
+  utmos: 0.0

benchmarks/DASB/LJSpeech/TTS/tokotron/hparams/train_dac.yaml

@@ -8,18 +8,23 @@ experiment_name: tokotron/dac
 # Seed needs to be set at top of yaml, before objects with parameters are made
 seed: 74443
 __set_seed: !apply:torch.manual_seed [!ref <seed>]
+run_name: !PLACEHOLDER
 output_folder: !ref results/<experiment_name>/<seed>
 save_folder: !ref <output_folder>/save
 train_log: !ref <output_folder>/train_log.txt
+testing: True # If set to True, the test evlaution is done, otherwise skipped.
+
 
 token_model_src: "facebook/encodec_24khz"
 g2p_src: flexthink/soundchoice-g2p
-vocoder_type: encodec
-vocoder_src: "charactr/vocos-encodec-24khz"
+
+# Model type
+representation_mode: discrete
 
 # Data files
-data_folder: !PLACEHOLDER # e.g., /path/to/LibriSpeech
-prepare_save_folder: !ref <data_folder>/prepared/dac
+data_folder: !PLACEHOLDER
+cached_data_folder: !PLACEHOLDER
+prepare_save_folder: !ref <cached_data_folder>
 pretrained_model_save_folder: !ref <prepare_save_folder>
 prepare_archive_path: null
 prepare_skip_ignore_folders: False
@@ -29,16 +34,27 @@ test_json: !ref <prepare_save_folder>/test.json
 frozen_split_path: null
 sample_path: null
 progress_folder: !ref <output_folder>/progress
-progress_archive: !ref <progress_folder>/progress.tar
 progress_current: !ref <progress_folder>/current
 progress_meta: !ref <progress_folder>/meta.yaml
 num_audio_samples: 32
 samples_interval: 5
 
-splits: ["train", "valid", "test"]
-split_ratio: [90, 5, 5]
 
+tokens_folder: !PLACEHOLDER  # Path to the folder where extracted tokens are saved.
+
+tokens_loader: !new:utils.tokens.TokensLoader
+    data_path: !ref <tokens_folder>
 
+token_model_kwargs:
+    n_quantizers: !ref <audio_tokens_per_step>
+
+splits: ["train", "valid", "test"]
+split_ratio: [90, 5, 5]
+ckpt_key: dwer
+ckpt_key_kind: min
+ckpt_keep: 2
+test_key: null
+test_key_kind: min
 ckpt_interval_minutes: 30 # save checkpoint every N min
 
 # Training parameters
@@ -61,7 +77,7 @@ bos_index: 0
 bos_width: 1
 
 # stages related parameters
-lr: 0.001
+lr: 0.001 # @orion_step1: --lr~"loguniform(0.00001,0.005)"
 lr_warmup_steps: 10000
 lr_annealing_mode: step
 guided_attention_weight: 50.0
@@ -85,33 +101,22 @@ model_bitrate: 8kbps
 
 # Label encoder
 label_encoder: !new:speechbrain.dataio.encoder.TextEncoder
-token_list_file_text: ./hparams/char_en.txt
-token_list_file_phn: ./hparams/arpabet.txt
+token_list_file_text: char_en.txt
+token_list_file_phn: arpabet.txt
 token_list_file: !apply:speechbrain.utils.hparams.choice
     value: !ref <input>
     choices:
         text: !ref <token_list_file_text>
         phonemes: !ref <token_list_file_phn>
 
 # Gate offset
-gate_offset: !apply:Tokotron.distance_diff_loss_ramp
+gate_offset: !apply:model.Tokotron.distance_diff_loss_ramp
     beta: !ref <gate_loss_beta>
     gamma: !ref <gate_loss_gamma>
     max_weight: !ref <gate_loss_max_weight>
 
 silence_padding: !ref <gate_offset>
 
-# Token model (pretrained)
-dac: !new:speechbrain.lobes.models.discrete.dac.DAC
-    sample_rate: !ref <model_sample_rate>
-    model_type: !ref <model_type>
-    model_bitrate: !ref <model_bitrate>
-    load_pretrained: True
-
-# Token model (pretrained)
-token_model: !new:Tokotron.DACFeatureExtractor
-    dac: !ref <dac>
-    n_quantizers: !ref <audio_tokens_per_step>
 
 # Dataloader options
 train_dataloader_opts:
@@ -143,20 +148,13 @@ sample_dataloader_opts:
         padding_kwargs:
             value: !ref <pad_index>
 
-extract_features_opts:
-    dataloader_opts:
-        batch_size: !ref <batch_size>
-    token_model: !ref <token_model>
-    sample_rate: !ref <sample_rate>
-    model_sample_rate: !ref <model_sample_rate>
-
 
 ####################### Model parameters ###########################
 # Transformer
 d_model: 512
 nhead: 4
-enc_num_layers: 6
-dec_num_layers: 12
+enc_num_layers: 6 # @orion_step1: --enc_num_layers~"choices([3, 6, 12])"
+dec_num_layers: 12 # @orion_step1: --dec_num_layers~"choices([3, 6, 12])"
 d_ffn: 2048
 transformer_dropout: 0.2
 target_dropout: 0.2
@@ -165,6 +163,7 @@ audio_num_tokens: 1024
 audio_emb_size: 1024
 audio_emb_freeze: False
 audio_emb_pretrained: False
+audio_token_offsets: False
 text_num_tokens: 39
 phn_num_tokens: 52
 input_num_tokens: !apply:speechbrain.utils.hparams.choice
@@ -178,7 +177,7 @@ attention_type: regularMHA
 
 ############################## models ################################
 
-model: !new:Tokotron.TokotronTransformerModel  # yamllint disable-line rule:line-length
+model: !new:model.Tokotron.TokotronTransformerModel  # yamllint disable-line rule:line-length
     input_num_tokens: !ref <input_num_tokens>
     audio_num_tokens: !ref <audio_num_tokens>
     audio_tokens_per_step: !ref <audio_tokens_per_step>
@@ -198,15 +197,23 @@ model: !new:Tokotron.TokotronTransformerModel  # yamllint disable-line rule:line
     max_audio_length: !ref <max_audio_length>
     infer_max_audio_length: !ref <infer_max_audio_length>
 
+tokenizer: !new:utils.tokenizer_interface.DACTokenizer
+    model_type: !ref <model_type>
+    model_bitrate: !ref <model_bitrate>
+    n_codebooks: !ref <audio_tokens_per_step>
+    load_pretrained: True
+    tag: latest
+
+
 modules:
     model: !ref <model>
-    dac: !ref <dac>
+    tokenizer: !ref <tokenizer>
 
 # define two optimizers here for two-stage training
 opt_class: !name:torch.optim.Adam
     lr: !ref <lr>
 
-compute_cost: !new:Tokotron.TokotronLoss
+compute_cost: !new:model.Tokotron.TokotronLoss
     guided_attention_weight: !ref <guided_attention_weight>
     guided_attention_sigma: !ref <guided_attention_sigma>
     gate_weight: !ref <gate_loss_weight>
@@ -226,10 +233,6 @@ checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
         lr_scheduler: !ref <lr_annealing>
         counter: !ref <epoch_counter>
 
-freezer: !new:preparation.Freezer
-    save_path: !ref <prepare_save_folder>
-    archive_path: !ref <prepare_archive_path>
-
 epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
     limit: !ref <number_of_epochs>