├── .gitattributes ├── LICENSE ├── Machine_Translation ├── .github │ ├── stale.yml │ └── workflows │ │ ├── build.yml │ │ └── build_wheels.yml ├── .gitignore ├── .gitmodules ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── docs │ ├── Makefile │ ├── _static │ │ └── theme_overrides.css │ ├── command_line_tools.rst │ ├── conf.py │ ├── criterions.rst │ ├── data.rst │ ├── docutils.conf │ ├── fairseq.gif │ ├── fairseq_logo.png │ ├── getting_started.rst │ ├── hydra_integration.md │ ├── index.rst │ ├── lr_scheduler.rst │ ├── make.bat │ ├── models.rst │ ├── modules.rst │ ├── optim.rst │ ├── overview.rst │ ├── requirements.txt │ ├── tasks.rst │ ├── tutorial_classifying_names.rst │ └── tutorial_simple_lstm.rst ├── examples │ ├── .gitignore │ ├── __init__.py │ ├── adaptive_span │ │ ├── README.md │ │ ├── __init__.py │ │ ├── adagrad_with_grad_clip.py │ │ ├── adaptive_span_attention.py │ │ ├── adaptive_span_loss.py │ │ ├── adaptive_span_model.py │ │ ├── adaptive_span_model_wrapper.py │ │ └── truncated_bptt_lm_task.py │ ├── backtranslation │ │ ├── README.md │ │ ├── deduplicate_lines.py │ │ ├── extract_bt_data.py │ │ ├── prepare-de-monolingual.sh │ │ ├── prepare-wmt18en2de.sh │ │ ├── sacrebleu.sh │ │ └── tokenized_bleu.sh │ ├── bart │ │ ├── README.glue.md │ │ ├── README.md │ │ ├── README.summarization.md │ │ └── summarize.py │ ├── byte_level_bpe │ │ ├── README.md │ │ ├── get_bitext.py │ │ ├── get_data.sh │ │ └── gru_transformer.py │ ├── camembert │ │ └── README.md │ ├── constrained_decoding │ │ ├── README.md │ │ ├── normalize.py │ │ └── tok.py │ ├── conv_seq2seq │ │ └── README.md │ ├── criss │ │ ├── README.md │ │ ├── download_and_preprocess_flores_test.sh │ │ ├── download_and_preprocess_tatoeba.sh │ │ ├── mining │ │ │ ├── mine.py │ │ │ └── mine_example.sh │ │ ├── save_encoder.py │ │ ├── sentence_retrieval │ │ │ ├── encoder_analysis.py │ │ │ └── sentence_retrieval_tatoeba.sh │ │ └── unsupervised_mt │ │ │ └── eval.sh │ ├── cross_lingual_language_model │ │ └── README.md │ ├── discriminative_reranking_nmt │ │ ├── README.md │ │ ├── __init__.py │ │ ├── config │ │ │ └── deen.yaml │ │ ├── criterions │ │ │ ├── __init__.py │ │ │ └── discriminative_reranking_criterion.py │ │ ├── drnmt_rerank.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ └── discriminative_reranking_model.py │ │ ├── scripts │ │ │ └── prep_data.py │ │ └── tasks │ │ │ ├── __init__.py │ │ │ └── discriminative_reranking_task.py │ ├── fast_noisy_channel │ │ ├── README.md │ │ ├── __init__.py │ │ ├── noisy_channel_beam_search.py │ │ ├── noisy_channel_sequence_generator.py │ │ └── noisy_channel_translation.py │ ├── flores101 │ │ ├── README.md │ │ └── flores_logo.png │ ├── fully_sharded_data_parallel │ │ └── README.md │ ├── gottbert │ │ └── README.md │ ├── hubert │ │ ├── README.md │ │ ├── config │ │ │ ├── decode │ │ │ │ ├── ax_sweep │ │ │ │ │ ├── ngram.yaml │ │ │ │ │ └── transformer.yaml │ │ │ │ ├── infer_fsqlm.yaml │ │ │ │ ├── infer_kenlm.yaml │ │ │ │ ├── infer_viterbi.yaml │ │ │ │ └── run │ │ │ │ │ ├── submitit_slurm.yaml │ │ │ │ │ └── submitit_slurm_8gpu.yaml │ │ │ ├── finetune │ │ │ │ ├── base_10h.yaml │ │ │ │ ├── ckpt │ │ │ │ │ └── it1.yaml │ │ │ │ ├── lm │ │ │ │ │ └── ls_4gram.yaml │ │ │ │ └── run │ │ │ │ │ └── submitit_reg.yaml │ │ │ └── pretrain │ │ │ │ ├── data │ │ │ │ ├── iter1.yaml │ │ │ │ └── iter2.yaml │ │ │ │ ├── hubert_base_librispeech.yaml │ │ │ │ ├── hubert_large_librivox.yaml │ │ │ │ ├── hubert_xlarge_librivox.yaml │ │ │ │ └── run │ │ │ │ └── submitit_reg.yaml │ │ ├── measure_teacher_quality.py │ │ ├── simple_kmeans │ │ │ ├── README.md │ │ │ ├── dump_hubert_feature.py │ │ │ ├── dump_hubert_feature_s2t.py │ │ │ ├── dump_km_label.py │ │ │ ├── dump_mfcc_feature.py │ │ │ ├── dump_w2v2_feature.py │ │ │ ├── feature_utils.py │ │ │ └── learn_kmeans.py │ │ └── update_ckpt.py │ ├── joint_alignment_translation │ │ ├── README.md │ │ └── prepare-wmt18en2de_no_norm_no_escape_no_agressive.sh │ ├── language_model │ │ ├── README.adaptive_inputs.md │ │ ├── README.conv.md │ │ ├── README.md │ │ └── prepare-wikitext-103.sh │ ├── laser │ │ ├── README.md │ │ └── laser_src │ │ │ ├── __init__.py │ │ │ ├── laser_lstm.py │ │ │ ├── laser_task.py │ │ │ ├── laser_transformer.py │ │ │ └── multitask_data_utils.py │ ├── latent_depth │ │ ├── README.md │ │ └── latent_depth_src │ │ │ ├── __init__.py │ │ │ ├── loss │ │ │ ├── __init__.py │ │ │ └── latent_depth.py │ │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── latent_multilingual_transformer.py │ │ │ └── latent_transformer.py │ │ │ ├── modules │ │ │ ├── __init__.py │ │ │ └── latent_layers.py │ │ │ └── multilingual_translation_latent_depth.py │ ├── layerdrop │ │ └── README.md │ ├── linformer │ │ ├── README.md │ │ └── linformer_src │ │ │ ├── __init__.py │ │ │ ├── models │ │ │ ├── __init__.py │ │ │ └── linformer_roberta.py │ │ │ └── modules │ │ │ ├── __init__.py │ │ │ ├── linformer_sentence_encoder.py │ │ │ ├── linformer_sentence_encoder_layer.py │ │ │ └── multihead_linear_attention.py │ ├── m2m_100 │ │ ├── README.md │ │ ├── install_dependecies.sh │ │ ├── process_data │ │ │ ├── clean_histogram.py │ │ │ ├── dedup_data.py │ │ │ └── remove_too_much_punc.py │ │ ├── tok.sh │ │ └── tokenizers │ │ │ ├── README.md │ │ │ ├── seg_ja.sh │ │ │ ├── seg_ko.sh │ │ │ ├── thirdparty │ │ │ └── .gitignore │ │ │ ├── tokenize_indic.py │ │ │ ├── tokenize_thai.py │ │ │ ├── tokenize_zh.py │ │ │ └── tokenizer_ar.sh │ ├── mbart │ │ └── README.md │ ├── megatron_11b │ │ ├── README.md │ │ └── detok.py │ ├── multilingual │ │ ├── ML50_langs.txt │ │ ├── README.md │ │ ├── data_scripts │ │ │ ├── README.md │ │ │ ├── binarize.py │ │ │ ├── check_iswlt_test_data.py │ │ │ ├── check_self_overlaps.py │ │ │ ├── check_valid_test_overlaps.py │ │ │ ├── dedup_all.py │ │ │ ├── download_ML50_v1.sh │ │ │ ├── download_af_xh.sh │ │ │ ├── download_flores_data.sh │ │ │ ├── download_iitb.sh │ │ │ ├── download_iwslt_and_extract.sh │ │ │ ├── download_lotus.sh │ │ │ ├── download_ted_and_extract.py │ │ │ ├── download_wat19_my.sh │ │ │ ├── download_wmt19_and_before.py │ │ │ ├── download_wmt20.sh │ │ │ ├── preprocess_ML50_v1.sh │ │ │ ├── remove_valid_test_in_train.py │ │ │ ├── requirement.txt │ │ │ └── utils │ │ │ │ ├── dedup.py │ │ │ │ ├── fasttext_multi_filter.py │ │ │ │ └── strip_sgm.sh │ │ ├── finetune_multilingual_model.sh │ │ ├── multilingual_fairseq_gen.sh │ │ └── train_multilingual_model.sh │ ├── noisychannel │ │ ├── README.md │ │ ├── __init__.py │ │ ├── rerank.py │ │ ├── rerank_generate.py │ │ ├── rerank_options.py │ │ ├── rerank_score_bw.py │ │ ├── rerank_score_lm.py │ │ ├── rerank_tune.py │ │ └── rerank_utils.py │ ├── nonautoregressive_translation │ │ ├── README.md │ │ └── scripts.md │ ├── paraphraser │ │ ├── README.md │ │ └── paraphrase.py │ ├── pay_less_attention_paper │ │ └── README.md │ ├── pointer_generator │ │ ├── README.md │ │ ├── README.xsum.md │ │ ├── pointer_generator_src │ │ │ ├── __init__.py │ │ │ └── transformer_pg.py │ │ ├── postprocess.py │ │ └── preprocess.py │ ├── quant_noise │ │ ├── README.md │ │ └── transformer_quantization_config.yaml │ ├── roberta │ │ ├── README.custom_classification.md │ │ ├── README.glue.md │ │ ├── README.md │ │ ├── README.pretraining.md │ │ ├── README.race.md │ │ ├── commonsense_qa │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── commonsense_qa_task.py │ │ │ └── download_cqa_data.sh │ │ ├── config │ │ │ ├── finetuning │ │ │ │ ├── cola.yaml │ │ │ │ ├── mnli.yaml │ │ │ │ ├── mrpc.yaml │ │ │ │ ├── qnli.yaml │ │ │ │ ├── qqp.yaml │ │ │ │ ├── rte.yaml │ │ │ │ ├── sst_2.yaml │ │ │ │ └── sts_b.yaml │ │ │ └── pretraining │ │ │ │ └── base.yaml │ │ ├── multiprocessing_bpe_encoder.py │ │ ├── preprocess_GLUE_tasks.sh │ │ ├── preprocess_RACE.py │ │ ├── preprocess_RACE.sh │ │ └── wsc │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── wsc_criterion.py │ │ │ ├── wsc_task.py │ │ │ └── wsc_utils.py │ ├── rxf │ │ ├── README.md │ │ ├── __init__.py │ │ └── rxf_src │ │ │ ├── __init__.py │ │ │ ├── label_smoothed_cross_entropy_r3f.py │ │ │ └── sentence_prediction_r3f.py │ ├── scaling_nmt │ │ └── README.md │ ├── simultaneous_translation │ │ ├── README.md │ │ ├── __init__.py │ │ ├── docs │ │ │ ├── ende-mma.md │ │ │ └── enja-waitk.md │ │ ├── eval │ │ │ └── agents │ │ │ │ └── simul_t2t_enja.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── convtransformer_simul_trans.py │ │ │ └── transformer_monotonic_attention.py │ │ ├── modules │ │ │ ├── __init__.py │ │ │ ├── fixed_pre_decision.py │ │ │ ├── monotonic_multihead_attention.py │ │ │ └── monotonic_transformer_layer.py │ │ ├── tests │ │ │ └── test_text_models.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── data_utils.py │ │ │ ├── functions.py │ │ │ ├── latency.py │ │ │ ├── monotonic_attention.py │ │ │ └── p_choose_strategy.py │ ├── speech_recognition │ │ ├── README.md │ │ ├── __init__.py │ │ ├── criterions │ │ │ ├── ASG_loss.py │ │ │ ├── __init__.py │ │ │ └── cross_entropy_acc.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── asr_dataset.py │ │ │ ├── collaters.py │ │ │ ├── data_utils.py │ │ │ └── replabels.py │ │ ├── datasets │ │ │ ├── asr_prep_json.py │ │ │ └── prepare-librispeech.sh │ │ ├── infer.py │ │ ├── kaldi │ │ │ ├── __init__.py │ │ │ ├── add-self-loop-simple.cc │ │ │ ├── config │ │ │ │ └── kaldi_initializer.yaml │ │ │ ├── kaldi_decoder.py │ │ │ └── kaldi_initializer.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── vggtransformer.py │ │ │ └── w2l_conv_glu_enc.py │ │ ├── new │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── conf │ │ │ │ ├── hydra │ │ │ │ │ └── sweeper │ │ │ │ │ │ └── ax.yaml │ │ │ │ └── infer.yaml │ │ │ ├── decoders │ │ │ │ ├── __init__.py │ │ │ │ ├── base_decoder.py │ │ │ │ ├── decoder.py │ │ │ │ ├── decoder_config.py │ │ │ │ ├── flashlight_decoder.py │ │ │ │ └── viterbi_decoder.py │ │ │ └── infer.py │ │ ├── tasks │ │ │ ├── __init__.py │ │ │ └── speech_recognition.py │ │ ├── utils │ │ │ └── wer_utils.py │ │ └── w2l_decoder.py │ ├── speech_text_joint_to_text │ │ ├── README.md │ │ ├── __init__.py │ │ ├── configs │ │ │ └── mustc_noise.list │ │ ├── criterions │ │ │ ├── __init__.py │ │ │ └── text_guide_cross_entropy_acc.py │ │ ├── docs │ │ │ ├── ende-mustc.md │ │ │ └── iwslt2021.md │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── s2t_dualinputtransformer.py │ │ │ └── s2t_dualinputxmtransformer.py │ │ ├── scripts │ │ │ └── g2p_encode.py │ │ └── tasks │ │ │ ├── __init__.py │ │ │ └── speech_text_joint.py │ ├── speech_to_text │ │ ├── README.md │ │ ├── data_utils.py │ │ ├── docs │ │ │ ├── covost_example.md │ │ │ ├── librispeech_example.md │ │ │ ├── mtedx_example.md │ │ │ ├── mustc_example.md │ │ │ └── simulst_mustc_example.md │ │ ├── prep_covost_data.py │ │ ├── prep_librispeech_data.py │ │ ├── prep_mtedx_data.py │ │ ├── prep_mustc_data.py │ │ ├── seg_mustc_data.py │ │ └── simultaneous_translation │ │ │ └── agents │ │ │ └── fairseq_simul_st_agent.py │ ├── stories │ │ └── README.md │ ├── translation │ │ ├── README.md │ │ ├── prepare-iwslt14.sh │ │ ├── prepare-iwslt17-multilingual.sh │ │ ├── prepare-wmt14en2de.sh │ │ └── prepare-wmt14en2fr.sh │ ├── translation_moe │ │ ├── README.md │ │ ├── score.py │ │ └── translation_moe_src │ │ │ ├── __init__.py │ │ │ ├── logsumexp_moe.py │ │ │ ├── mean_pool_gating_network.py │ │ │ └── translation_moe.py │ ├── truncated_bptt │ │ ├── README.md │ │ ├── __init__.py │ │ ├── transformer_xl_model.py │ │ └── truncated_bptt_lm_task.py │ ├── unsupervised_quality_estimation │ │ ├── README.md │ │ ├── aggregate_scores.py │ │ ├── meteor.py │ │ └── repeat_lines.py │ ├── wav2vec │ │ ├── README.md │ │ ├── __init__.py │ │ ├── config │ │ │ ├── finetuning │ │ │ │ ├── base_100h.yaml │ │ │ │ ├── base_10h.yaml │ │ │ │ ├── base_10m.yaml │ │ │ │ ├── base_1h.yaml │ │ │ │ ├── base_960h.yaml │ │ │ │ ├── vox_100h.yaml │ │ │ │ ├── vox_10h.yaml │ │ │ │ ├── vox_10m.yaml │ │ │ │ ├── vox_1h.yaml │ │ │ │ └── vox_960h.yaml │ │ │ └── pretraining │ │ │ │ ├── wav2vec2_base_librispeech.yaml │ │ │ │ ├── wav2vec2_large_librivox.yaml │ │ │ │ ├── wav2vec2_large_librivox_tpu-pod.yaml │ │ │ │ └── wav2vec2_large_librivox_tpu.yaml │ │ ├── libri_labels.py │ │ ├── scripts │ │ │ └── binarize_manifest.sh │ │ ├── unsupervised │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── config │ │ │ │ ├── finetuning │ │ │ │ │ └── w2v_finetune.yaml │ │ │ │ ├── gan │ │ │ │ │ └── w2vu.yaml │ │ │ │ ├── generate │ │ │ │ │ └── viterbi.yaml │ │ │ │ ├── timit_matched │ │ │ │ │ ├── test.uid │ │ │ │ │ ├── train.uid │ │ │ │ │ ├── train_text.uid │ │ │ │ │ └── valid.uid │ │ │ │ └── timit_unmatched │ │ │ │ │ ├── test.uid │ │ │ │ │ ├── train.uid │ │ │ │ │ ├── train_text.uid │ │ │ │ │ └── valid.uid │ │ │ ├── data │ │ │ │ ├── __init__.py │ │ │ │ ├── extracted_features_dataset.py │ │ │ │ └── random_input_dataset.py │ │ │ ├── kaldi_self_train │ │ │ │ ├── README.md │ │ │ │ └── st │ │ │ │ │ ├── cmd.sh │ │ │ │ │ ├── decode_phone.sh │ │ │ │ │ ├── decode_word_step1.sh │ │ │ │ │ ├── decode_word_step2.sh │ │ │ │ │ ├── local │ │ │ │ │ ├── copy_aligned_text.py │ │ │ │ │ ├── decode.sh │ │ │ │ │ ├── prepare_data_from_w2v.py │ │ │ │ │ ├── prepare_lang.sh │ │ │ │ │ ├── prepare_lang_word.sh │ │ │ │ │ ├── prepare_lm.sh │ │ │ │ │ ├── score.sh │ │ │ │ │ ├── show_wer.sh │ │ │ │ │ ├── train_subset_lgbeam.sh │ │ │ │ │ ├── unsup_select.py │ │ │ │ │ ├── unsup_select_decode.sh │ │ │ │ │ └── unsup_select_decode_word.sh │ │ │ │ │ ├── path.sh │ │ │ │ │ ├── steps │ │ │ │ │ ├── steps_gan │ │ │ │ │ ├── train_deltas.sh │ │ │ │ │ ├── train_lda_mllt.sh │ │ │ │ │ └── train_sat.sh │ │ │ │ │ ├── train.sh │ │ │ │ │ └── utils │ │ │ ├── models │ │ │ │ ├── __init__.py │ │ │ │ └── wav2vec_u.py │ │ │ ├── scripts │ │ │ │ ├── apply_pca.py │ │ │ │ ├── copy_labels.py │ │ │ │ ├── filter_lexicon.py │ │ │ │ ├── filter_tsv.py │ │ │ │ ├── g2p_wrd_to_phn.py │ │ │ │ ├── ltr_to_wrd.py │ │ │ │ ├── mean_pool.py │ │ │ │ ├── merge_clusters.py │ │ │ │ ├── normalize_and_filter_text.py │ │ │ │ ├── normalize_text.py │ │ │ │ ├── pca.py │ │ │ │ ├── phonemize_with_sil.py │ │ │ │ ├── prepare_audio.sh │ │ │ │ ├── prepare_text.sh │ │ │ │ ├── prepare_timit.sh │ │ │ │ ├── remove_silence.py │ │ │ │ ├── vads.py │ │ │ │ ├── wav2vec_apply_cluster_faiss.py │ │ │ │ ├── wav2vec_cluster_faiss.py │ │ │ │ ├── wav2vec_extract_features.py │ │ │ │ ├── wer.py │ │ │ │ └── wrd_to_ltr.py │ │ │ ├── tasks │ │ │ │ ├── __init__.py │ │ │ │ └── unpaired_audio_text.py │ │ │ └── w2vu_generate.py │ │ ├── vq-wav2vec_featurize.py │ │ ├── wav2vec_featurize.py │ │ └── wav2vec_manifest.py │ ├── wmt19 │ │ └── README.md │ ├── wmt20 │ │ └── README.md │ └── xlmr │ │ └── README.md ├── fairseq │ ├── __init__.py │ ├── benchmark │ │ ├── __init__.py │ │ ├── dummy_dataset.py │ │ ├── dummy_lm.py │ │ ├── dummy_masked_lm.py │ │ ├── dummy_model.py │ │ └── dummy_mt.py │ ├── binarizer.py │ ├── checkpoint_utils.py │ ├── clib │ │ ├── cuda │ │ │ ├── ngram_repeat_block_cuda.cpp │ │ │ └── ngram_repeat_block_cuda_kernel.cu │ │ ├── libbase │ │ │ └── balanced_assignment.cpp │ │ ├── libbleu │ │ │ ├── libbleu.cpp │ │ │ └── module.cpp │ │ ├── libnat │ │ │ └── edit_dist.cpp │ │ └── libnat_cuda │ │ │ ├── binding.cpp │ │ │ ├── edit_dist.cu │ │ │ └── edit_dist.h │ ├── config │ │ ├── __init__.py │ │ ├── config.yaml │ │ └── model │ │ │ ├── transformer_lm │ │ │ ├── transformer_lm_baevski_gbw.yaml │ │ │ ├── transformer_lm_baevski_wiki103.yaml │ │ │ ├── transformer_lm_big.yaml │ │ │ ├── transformer_lm_gbw.yaml │ │ │ ├── transformer_lm_gpt.yaml │ │ │ ├── transformer_lm_gpt2_big.yaml │ │ │ ├── transformer_lm_gpt2_medium.yaml │ │ │ ├── transformer_lm_gpt2_small.yaml │ │ │ └── transformer_lm_wiki103.yaml │ │ │ ├── wav2vec │ │ │ └── vq_wav2vec_gumbel.yaml │ │ │ └── wav2vec2 │ │ │ ├── wav2vec2_base.yaml │ │ │ └── wav2vec2_large.yaml │ ├── criterions │ │ ├── __init__.py │ │ ├── adaptive_loss.py │ │ ├── additional_loss.py │ │ ├── composite_loss.py │ │ ├── cross_entropy.py │ │ ├── ctc.py │ │ ├── fairseq_criterion.py │ │ ├── hubert_criterion.py │ │ ├── label_smoothed_cross_entropy.py │ │ ├── label_smoothed_cross_entropy_latency_augmented.py │ │ ├── label_smoothed_cross_entropy_with_alignment.py │ │ ├── legacy_masked_lm.py │ │ ├── masked_lm.py │ │ ├── model_criterion.py │ │ ├── nat_loss.py │ │ ├── sentence_prediction.py │ │ ├── sentence_ranking.py │ │ └── wav2vec_criterion.py │ ├── data │ │ ├── __init__.py │ │ ├── add_target_dataset.py │ │ ├── append_token_dataset.py │ │ ├── audio │ │ │ ├── __init__.py │ │ │ ├── audio_utils.py │ │ │ ├── feature_transforms │ │ │ │ ├── __init__.py │ │ │ │ ├── global_cmvn.py │ │ │ │ ├── specaugment.py │ │ │ │ └── utterance_cmvn.py │ │ │ ├── hubert_dataset.py │ │ │ ├── multi_modality_dataset.py │ │ │ ├── raw_audio_dataset.py │ │ │ ├── speech_to_text_dataset.py │ │ │ └── speech_to_text_joint_dataset.py │ │ ├── backtranslation_dataset.py │ │ ├── base_wrapper_dataset.py │ │ ├── bucket_pad_length_dataset.py │ │ ├── colorize_dataset.py │ │ ├── concat_dataset.py │ │ ├── concat_sentences_dataset.py │ │ ├── data_utils.py │ │ ├── data_utils_fast.pyx │ │ ├── denoising_dataset.py │ │ ├── dictionary.py │ │ ├── encoders │ │ │ ├── __init__.py │ │ │ ├── byte_bpe.py │ │ │ ├── byte_utils.py │ │ │ ├── bytes.py │ │ │ ├── characters.py │ │ │ ├── fastbpe.py │ │ │ ├── gpt2_bpe.py │ │ │ ├── gpt2_bpe_utils.py │ │ │ ├── hf_bert_bpe.py │ │ │ ├── hf_byte_bpe.py │ │ │ ├── moses_tokenizer.py │ │ │ ├── nltk_tokenizer.py │ │ │ ├── sentencepiece_bpe.py │ │ │ ├── space_tokenizer.py │ │ │ ├── subword_nmt_bpe.py │ │ │ └── utils.py │ │ ├── fairseq_dataset.py │ │ ├── fasta_dataset.py │ │ ├── id_dataset.py │ │ ├── indexed_dataset.py │ │ ├── iterators.py │ │ ├── language_pair_dataset.py │ │ ├── legacy │ │ │ ├── __init__.py │ │ │ ├── block_pair_dataset.py │ │ │ ├── masked_lm_dataset.py │ │ │ └── masked_lm_dictionary.py │ │ ├── list_dataset.py │ │ ├── lm_context_window_dataset.py │ │ ├── lru_cache_dataset.py │ │ ├── mask_tokens_dataset.py │ │ ├── monolingual_dataset.py │ │ ├── multi_corpus_dataset.py │ │ ├── multi_corpus_sampled_dataset.py │ │ ├── multilingual │ │ │ ├── __init__.py │ │ │ ├── multilingual_data_manager.py │ │ │ ├── multilingual_utils.py │ │ │ ├── sampled_multi_dataset.py │ │ │ ├── sampled_multi_epoch_dataset.py │ │ │ └── sampling_method.py │ │ ├── nested_dictionary_dataset.py │ │ ├── noising.py │ │ ├── num_samples_dataset.py │ │ ├── numel_dataset.py │ │ ├── offset_tokens_dataset.py │ │ ├── pad_dataset.py │ │ ├── plasma_utils.py │ │ ├── prepend_dataset.py │ │ ├── prepend_token_dataset.py │ │ ├── raw_label_dataset.py │ │ ├── replace_dataset.py │ │ ├── resampling_dataset.py │ │ ├── roll_dataset.py │ │ ├── round_robin_zip_datasets.py │ │ ├── shorten_dataset.py │ │ ├── sort_dataset.py │ │ ├── strip_token_dataset.py │ │ ├── subsample_dataset.py │ │ ├── text_compressor.py │ │ ├── token_block_dataset.py │ │ ├── token_block_utils_fast.pyx │ │ ├── transform_eos_dataset.py │ │ └── transform_eos_lang_pair_dataset.py │ ├── dataclass │ │ ├── __init__.py │ │ ├── configs.py │ │ ├── constants.py │ │ ├── initialize.py │ │ └── utils.py │ ├── distributed │ │ ├── __init__.py │ │ ├── distributed_timeout_wrapper.py │ │ ├── fully_sharded_data_parallel.py │ │ ├── legacy_distributed_data_parallel.py │ │ ├── module_proxy_wrapper.py │ │ ├── tpu_distributed_data_parallel.py │ │ └── utils.py │ ├── file_chunker_utils.py │ ├── file_io.py │ ├── file_utils.py │ ├── hub_utils.py │ ├── incremental_decoding_utils.py │ ├── iterative_refinement_generator.py │ ├── logging │ │ ├── __init__.py │ │ ├── meters.py │ │ ├── metrics.py │ │ └── progress_bar.py │ ├── model_parallel │ │ ├── __init__.py │ │ ├── criterions │ │ │ ├── __init__.py │ │ │ └── vocab_parallel_cross_entropy.py │ │ ├── megatron_trainer.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── pipeline_parallel_transformer │ │ │ │ ├── __init__.py │ │ │ │ ├── layers.py │ │ │ │ └── model.py │ │ │ ├── roberta │ │ │ │ ├── __init__.py │ │ │ │ └── model.py │ │ │ ├── transformer.py │ │ │ └── transformer_lm.py │ │ └── modules │ │ │ ├── __init__.py │ │ │ ├── multihead_attention.py │ │ │ └── transformer_layer.py │ ├── models │ │ ├── __init__.py │ │ ├── bart │ │ │ ├── __init__.py │ │ │ ├── hub_interface.py │ │ │ └── model.py │ │ ├── composite_encoder.py │ │ ├── distributed_fairseq_model.py │ │ ├── fairseq_decoder.py │ │ ├── fairseq_encoder.py │ │ ├── fairseq_incremental_decoder.py │ │ ├── fairseq_model.py │ │ ├── fconv.py │ │ ├── fconv_lm.py │ │ ├── fconv_self_att.py │ │ ├── hubert │ │ │ ├── __init__.py │ │ │ ├── hubert.py │ │ │ └── hubert_asr.py │ │ ├── huggingface │ │ │ ├── __init__.py │ │ │ └── hf_gpt2.py │ │ ├── lightconv.py │ │ ├── lightconv_lm.py │ │ ├── lstm.py │ │ ├── lstm_lm.py │ │ ├── masked_lm.py │ │ ├── model_utils.py │ │ ├── multilingual_transformer.py │ │ ├── nat │ │ │ ├── __init__.py │ │ │ ├── cmlm_transformer.py │ │ │ ├── fairseq_nat_model.py │ │ │ ├── insertion_transformer.py │ │ │ ├── iterative_nonautoregressive_transformer.py │ │ │ ├── levenshtein_transformer.py │ │ │ ├── levenshtein_utils.py │ │ │ ├── nat_crf_transformer.py │ │ │ ├── nonautoregressive_ensembles.py │ │ │ └── nonautoregressive_transformer.py │ │ ├── roberta │ │ │ ├── __init__.py │ │ │ ├── alignment_utils.py │ │ │ ├── enc_dec.py │ │ │ ├── hub_interface.py │ │ │ ├── model.py │ │ │ ├── model_camembert.py │ │ │ ├── model_gottbert.py │ │ │ └── model_xlmr.py │ │ ├── speech_to_text │ │ │ ├── __init__.py │ │ │ ├── berard.py │ │ │ ├── convtransformer.py │ │ │ ├── modules │ │ │ │ ├── augmented_memory_attention.py │ │ │ │ └── emformer.py │ │ │ ├── s2t_transformer.py │ │ │ ├── utils.py │ │ │ └── xm_transformer.py │ │ ├── transformer.py │ │ ├── transformer │ │ │ ├── __init__.py │ │ │ ├── transformer_base.py │ │ │ ├── transformer_config.py │ │ │ ├── transformer_decoder.py │ │ │ ├── transformer_encoder.py │ │ │ └── transformer_legacy.py │ │ ├── transformer_align.py │ │ ├── transformer_from_pretrained_xlm.py │ │ ├── transformer_lm.py │ │ └── wav2vec │ │ │ ├── __init__.py │ │ │ ├── wav2vec.py │ │ │ ├── wav2vec2.py │ │ │ └── wav2vec2_asr.py │ ├── modules │ │ ├── __init__.py │ │ ├── adaptive_input.py │ │ ├── adaptive_softmax.py │ │ ├── base_layer.py │ │ ├── beamable_mm.py │ │ ├── character_token_embedder.py │ │ ├── checkpoint_activations.py │ │ ├── conv_tbc.py │ │ ├── cross_entropy.py │ │ ├── cuda_utils.cu │ │ ├── downsampled_multihead_attention.py │ │ ├── dynamic_convolution.py │ │ ├── dynamic_crf_layer.py │ │ ├── dynamicconv_layer │ │ │ ├── __init__.py │ │ │ ├── cuda_function_gen.py │ │ │ ├── dynamicconv_cuda.cpp │ │ │ ├── dynamicconv_cuda.cuh │ │ │ ├── dynamicconv_cuda_kernel.cu │ │ │ ├── dynamicconv_layer.py │ │ │ ├── dynamiconv_cpu.cpp │ │ │ └── setup.py │ │ ├── fairseq_dropout.py │ │ ├── fp32_group_norm.py │ │ ├── gelu.py │ │ ├── grad_multiply.py │ │ ├── gumbel_vector_quantizer.py │ │ ├── kmeans_attention.py │ │ ├── kmeans_vector_quantizer.py │ │ ├── layer_drop.py │ │ ├── layer_norm.py │ │ ├── learned_positional_embedding.py │ │ ├── lightconv_layer │ │ │ ├── __init__.py │ │ │ ├── cuda_function_gen.py │ │ │ ├── lightconv_cuda.cpp │ │ │ ├── lightconv_cuda.cuh │ │ │ ├── lightconv_cuda_kernel.cu │ │ │ ├── lightconv_layer.py │ │ │ └── setup.py │ │ ├── lightweight_convolution.py │ │ ├── linearized_convolution.py │ │ ├── memory_component.py │ │ ├── multihead_attention.py │ │ ├── positional_embedding.py │ │ ├── quant_noise.py │ │ ├── quantization │ │ │ ├── __init__.py │ │ │ ├── pq │ │ │ │ ├── __init__.py │ │ │ │ ├── em.py │ │ │ │ ├── modules │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── qconv.py │ │ │ │ │ ├── qemb.py │ │ │ │ │ └── qlinear.py │ │ │ │ ├── pq.py │ │ │ │ └── utils.py │ │ │ ├── quantization_options.py │ │ │ └── scalar │ │ │ │ ├── __init__.py │ │ │ │ ├── modules │ │ │ │ ├── __init__.py │ │ │ │ ├── qact.py │ │ │ │ ├── qconv.py │ │ │ │ ├── qemb.py │ │ │ │ └── qlinear.py │ │ │ │ ├── ops.py │ │ │ │ └── utils.py │ │ ├── same_pad.py │ │ ├── scalar_bias.py │ │ ├── sinusoidal_positional_embedding.py │ │ ├── sparse_multihead_attention.py │ │ ├── sparse_transformer_sentence_encoder.py │ │ ├── sparse_transformer_sentence_encoder_layer.py │ │ ├── transformer_layer.py │ │ ├── transformer_sentence_encoder.py │ │ ├── transformer_sentence_encoder_layer.py │ │ ├── transpose_last.py │ │ ├── unfold.py │ │ └── vggblock.py │ ├── nan_detector.py │ ├── ngram_repeat_block.py │ ├── optim │ │ ├── __init__.py │ │ ├── adadelta.py │ │ ├── adafactor.py │ │ ├── adagrad.py │ │ ├── adam.py │ │ ├── adamax.py │ │ ├── amp_optimizer.py │ │ ├── bmuf.py │ │ ├── composite.py │ │ ├── cpu_adam.py │ │ ├── dynamic_loss_scaler.py │ │ ├── fairseq_optimizer.py │ │ ├── fp16_optimizer.py │ │ ├── fused_adam.py │ │ ├── fused_lamb.py │ │ ├── lr_scheduler │ │ │ ├── __init__.py │ │ │ ├── cosine_lr_scheduler.py │ │ │ ├── fairseq_lr_scheduler.py │ │ │ ├── fixed_schedule.py │ │ │ ├── inverse_square_root_schedule.py │ │ │ ├── manual_lr_scheduler.py │ │ │ ├── pass_through.py │ │ │ ├── polynomial_decay_schedule.py │ │ │ ├── reduce_lr_on_plateau.py │ │ │ ├── tri_stage_lr_scheduler.py │ │ │ └── triangular_lr_scheduler.py │ │ ├── nag.py │ │ ├── sgd.py │ │ └── shard.py │ ├── options.py │ ├── pdb.py │ ├── quantization_utils.py │ ├── registry.py │ ├── scoring │ │ ├── __init__.py │ │ ├── bleu.py │ │ ├── chrf.py │ │ ├── tokenizer.py │ │ └── wer.py │ ├── search.py │ ├── sequence_generator.py │ ├── sequence_scorer.py │ ├── tasks │ │ ├── __init__.py │ │ ├── audio_finetuning.py │ │ ├── audio_pretraining.py │ │ ├── cross_lingual_lm.py │ │ ├── denoising.py │ │ ├── fairseq_task.py │ │ ├── hubert_pretraining.py │ │ ├── language_modeling.py │ │ ├── legacy_masked_lm.py │ │ ├── masked_lm.py │ │ ├── multilingual_denoising.py │ │ ├── multilingual_masked_lm.py │ │ ├── multilingual_translation.py │ │ ├── online_backtranslation.py │ │ ├── semisupervised_translation.py │ │ ├── sentence_prediction.py │ │ ├── sentence_ranking.py │ │ ├── simultaneous_translation.py │ │ ├── speech_to_text.py │ │ ├── translation.py │ │ ├── translation_from_pretrained_bart.py │ │ ├── translation_from_pretrained_xlm.py │ │ ├── translation_lev.py │ │ └── translation_multi_simple_epoch.py │ ├── token_generation_constraints.py │ ├── tokenizer.py │ ├── trainer.py │ ├── utils.py │ └── version.txt ├── fairseq_cli │ ├── __init__.py │ ├── eval_lm.py │ ├── generate.py │ ├── hydra_train.py │ ├── interactive.py │ ├── preprocess.py │ ├── score.py │ ├── train.py │ └── validate.py ├── hubconf.py ├── pyproject.toml ├── scripts │ ├── __init__.py │ ├── average_checkpoints.py │ ├── build_sym_alignment.py │ ├── compare_namespaces.py │ ├── compound_split_bleu.sh │ ├── constraints │ │ ├── extract.py │ │ └── validate.py │ ├── convert_dictionary.lua │ ├── convert_model.lua │ ├── count_docs.py │ ├── read_binarized.py │ ├── rm_pt.py │ ├── sacrebleu.sh │ ├── shard_docs.py │ ├── split_train_valid_docs.py │ ├── spm_decode.py │ ├── spm_encode.py │ ├── spm_train.py │ └── test_fsdp.sh ├── setup.py ├── tests │ ├── __init__.py │ ├── distributed │ │ ├── __init__.py │ │ ├── test_bmuf.py │ │ ├── test_distributed_timeout_wrapper.py │ │ ├── test_module_proxy_wrapper.py │ │ ├── test_utils.py │ │ └── utils.py │ ├── gpu │ │ ├── __init__.py │ │ ├── test_binaries_gpu.py │ │ └── transformer_quantization_config.yaml │ ├── speech_recognition │ │ ├── __init__.py │ │ ├── asr_test_base.py │ │ ├── test_collaters.py │ │ ├── test_cross_entropy.py │ │ ├── test_data_utils.py │ │ └── test_vggtransformer.py │ ├── test_activation_checkpointing.py │ ├── test_amp_optimizer.py │ ├── test_average_checkpoints.py │ ├── test_backtranslation_dataset.py │ ├── test_binaries.py │ ├── test_character_token_embedder.py │ ├── test_checkpoint_utils.py │ ├── test_concat_dataset.py │ ├── test_constraints.py │ ├── test_convtbc.py │ ├── test_data_utils.py │ ├── test_dataclass_utils.py │ ├── test_dataset.py │ ├── test_dictionary.py │ ├── test_export.py │ ├── test_file_chunker_utils.py │ ├── test_file_io.py │ ├── test_fp16_optimizer.py │ ├── test_inference_dropout.py │ ├── test_iopath.py │ ├── test_iterators.py │ ├── test_label_smoothing.py │ ├── test_lm_context_window.py │ ├── test_lstm_jitable.py │ ├── test_memory_efficient_fp16.py │ ├── test_metrics.py │ ├── test_multi_corpus_dataset.py │ ├── test_multi_corpus_sampled_dataset.py │ ├── test_multihead_attention.py │ ├── test_noising.py │ ├── test_online_backtranslation.py │ ├── test_plasma_utils.py │ ├── test_reproducibility.py │ ├── test_resampling_dataset.py │ ├── test_roberta.py │ ├── test_sequence_generator.py │ ├── test_sequence_scorer.py │ ├── test_sparse_multihead_attention.py │ ├── test_token_block_dataset.py │ ├── test_train.py │ ├── test_transformer.py │ ├── test_utils.py │ ├── test_valid_subset_checks.py │ └── utils.py └── train.py ├── README.md └── Speech_Recognition ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── egs └── aishell │ └── asr │ ├── RESULTS.md │ ├── cmd.sh │ ├── conf │ ├── decode.yaml │ ├── decode_memory_transformer.yaml │ ├── decode_sync_transformer.yaml │ ├── fbank.conf │ ├── gpu.conf │ ├── lm.yaml │ ├── pitch.conf │ ├── plot_compressive_transformer.yaml │ ├── queue.conf │ ├── slurm.conf │ ├── specaug.yaml │ ├── testing.yaml │ ├── train.yaml │ ├── train_memory_transformer.yaml │ ├── train_synctransformer.yaml │ ├── train_transducer.yaml │ ├── train_transformer.yaml │ └── tuning │ │ ├── decode_pytorch_transformer.yaml │ │ ├── decode_rnn.yaml │ │ ├── train_pytorch_transformer.yaml │ │ └── train_rnn.yaml │ ├── dec_com.sh │ ├── dec_sync.sh │ ├── local │ ├── aishell_data_prep.sh │ ├── download_and_untar.sh │ ├── make_voxceleb1.pl │ ├── make_voxceleb1_v2.pl │ ├── make_voxceleb2.pl │ ├── nnet3 │ │ └── xvector │ │ │ ├── prepare_feats_for_egs.sh │ │ │ ├── run_xvector.sh │ │ │ └── tuning │ │ │ └── run_xvector_1a.sh │ └── prepare_for_eer.py │ ├── path.sh │ ├── run.sh │ ├── run_with_memory.sh │ ├── sid │ ├── compute_min_dcf.py │ ├── compute_vad_decision.sh │ ├── compute_vad_decision_gmm.sh │ ├── extract_ivectors.sh │ ├── extract_ivectors_dnn.sh │ ├── gender_id.sh │ ├── init_full_ubm_from_dnn.sh │ ├── music_id.sh │ ├── nnet2 │ │ ├── get_egs2.sh │ │ ├── get_lda.sh │ │ └── train_multisplice_accel2.sh │ ├── nnet3 │ │ └── xvector │ │ │ ├── allocate_egs.py │ │ │ ├── extract_xvectors.sh │ │ │ └── get_egs.sh │ ├── train_diag_ubm.sh │ ├── train_full_ubm.sh │ ├── train_ivector_extractor.sh │ └── train_ivector_extractor_dnn.sh │ ├── steps │ ├── align_basis_fmllr.sh │ ├── align_basis_fmllr_lats.sh │ ├── align_fmllr.sh │ ├── align_fmllr_lats.sh │ ├── align_lvtln.sh │ ├── align_raw_fmllr.sh │ ├── align_sgmm2.sh │ ├── align_si.sh │ ├── append_feats.sh │ ├── best_path_weights.sh │ ├── chain │ │ ├── build_tree.sh │ │ ├── build_tree_multiple_sources.sh │ │ ├── e2e │ │ │ ├── README.txt │ │ │ ├── compute_biphone_stats.py │ │ │ ├── get_egs_e2e.sh │ │ │ ├── prepare_e2e.sh │ │ │ ├── text_to_phones.py │ │ │ └── train_e2e.py │ │ ├── gen_topo.pl │ │ ├── gen_topo.py │ │ ├── gen_topo2.py │ │ ├── gen_topo3.py │ │ ├── gen_topo4.py │ │ ├── gen_topo5.py │ │ ├── gen_topo_orig.py │ │ ├── get_egs.sh │ │ ├── get_phone_post.sh │ │ ├── make_weighted_den_fst.sh │ │ ├── multilingual │ │ │ └── combine_egs.sh │ │ ├── train.py │ │ └── train_tdnn.sh │ ├── cleanup │ │ ├── clean_and_segment_data.sh │ │ ├── clean_and_segment_data_nnet3.sh │ │ ├── combine_short_segments.py │ │ ├── create_segments_from_ctm.pl │ │ ├── debug_lexicon.sh │ │ ├── decode_fmllr_segmentation.sh │ │ ├── decode_segmentation.sh │ │ ├── decode_segmentation_nnet3.sh │ │ ├── find_bad_utts.sh │ │ ├── find_bad_utts_nnet.sh │ │ ├── internal │ │ │ ├── align_ctm_ref.py │ │ │ ├── compute_tf_idf.py │ │ │ ├── ctm_to_text.pl │ │ │ ├── get_ctm_edits.py │ │ │ ├── get_non_scored_words.py │ │ │ ├── get_pron_stats.py │ │ │ ├── make_one_biased_lm.py │ │ │ ├── modify_ctm_edits.py │ │ │ ├── resolve_ctm_edits_overlaps.py │ │ │ ├── retrieve_similar_docs.py │ │ │ ├── segment_ctm_edits.py │ │ │ ├── segment_ctm_edits_mild.py │ │ │ ├── split_text_into_docs.pl │ │ │ ├── stitch_documents.py │ │ │ ├── taint_ctm_edits.py │ │ │ └── tf_idf.py │ │ ├── lattice_oracle_align.sh │ │ ├── make_biased_lm_graphs.sh │ │ ├── make_biased_lms.py │ │ ├── make_segmentation_data_dir.sh │ │ ├── make_segmentation_graph.sh │ │ ├── make_utterance_fsts.pl │ │ ├── make_utterance_graph.sh │ │ ├── segment_long_utterances.sh │ │ ├── segment_long_utterances_nnet3.sh │ │ └── split_long_utterance.sh │ ├── combine_ali_dirs.sh │ ├── combine_lat_dirs.sh │ ├── compare_alignments.sh │ ├── compute_cmvn_stats.sh │ ├── compute_vad_decision.sh │ ├── conf │ │ ├── append_eval_to_ctm.py │ │ ├── append_prf_to_ctm.py │ │ ├── apply_calibration.sh │ │ ├── convert_ctm_to_tra.py │ │ ├── get_ctm_conf.sh │ │ ├── lattice_depth_per_frame.sh │ │ ├── parse_arpa_unigrams.py │ │ ├── prepare_calibration_data.py │ │ ├── prepare_word_categories.py │ │ └── train_calibration.sh │ ├── copy_ali_dir.sh │ ├── copy_lat_dir.sh │ ├── data │ │ ├── augment_data_dir.py │ │ ├── data_dir_manipulation_lib.py │ │ ├── make_musan.py │ │ ├── make_musan.sh │ │ └── reverberate_data_dir.py │ ├── decode.sh │ ├── decode_basis_fmllr.sh │ ├── decode_biglm.sh │ ├── decode_combine.sh │ ├── decode_fmllr.sh │ ├── decode_fmllr_extra.sh │ ├── decode_fmmi.sh │ ├── decode_fromlats.sh │ ├── decode_lvtln.sh │ ├── decode_nnet.sh │ ├── decode_nolats.sh │ ├── decode_raw_fmllr.sh │ ├── decode_sgmm2.sh │ ├── decode_sgmm2_fromlats.sh │ ├── decode_sgmm2_rescore.sh │ ├── decode_sgmm2_rescore_project.sh │ ├── decode_si.sh │ ├── decode_with_map.sh │ ├── diagnostic │ │ ├── analyze_alignments.sh │ │ ├── analyze_lats.sh │ │ ├── analyze_lattice_depth_stats.py │ │ └── analyze_phone_length_stats.py │ ├── dict │ │ ├── apply_g2p.sh │ │ ├── apply_g2p_phonetisaurus.sh │ │ ├── apply_lexicon_edits.py │ │ ├── get_pron_stats.py │ │ ├── internal │ │ │ ├── get_subsegments.py │ │ │ ├── prune_pron_candidates.py │ │ │ └── sum_arc_info.py │ │ ├── learn_lexicon_bayesian.sh │ │ ├── learn_lexicon_greedy.sh │ │ ├── merge_learned_lexicons.py │ │ ├── prons_to_lexicon.py │ │ ├── prune_pron_candidates.py │ │ ├── select_prons_bayesian.py │ │ ├── select_prons_greedy.py │ │ ├── train_g2p.sh │ │ └── train_g2p_phonetisaurus.sh │ ├── get_ctm.sh │ ├── get_ctm_conf.sh │ ├── get_ctm_conf_fast.sh │ ├── get_ctm_fast.sh │ ├── get_fmllr_basis.sh │ ├── get_lexicon_probs.sh │ ├── get_prons.sh │ ├── get_train_ctm.sh │ ├── info │ │ ├── chain_dir_info.pl │ │ ├── gmm_dir_info.pl │ │ ├── nnet2_dir_info.pl │ │ ├── nnet3_dir_info.pl │ │ └── nnet3_disc_dir_info.pl │ ├── libs │ │ ├── __init__.py │ │ ├── common.py │ │ └── nnet3 │ │ │ ├── __init__.py │ │ │ ├── report │ │ │ ├── __init__.py │ │ │ └── log_parse.py │ │ │ ├── train │ │ │ ├── __init__.py │ │ │ ├── chain_objf │ │ │ │ ├── __init__.py │ │ │ │ └── acoustic_model.py │ │ │ ├── common.py │ │ │ ├── dropout_schedule.py │ │ │ └── frame_level_objf │ │ │ │ ├── __init__.py │ │ │ │ ├── acoustic_model.py │ │ │ │ ├── common.py │ │ │ │ └── raw_model.py │ │ │ └── xconfig │ │ │ ├── __init__.py │ │ │ ├── attention.py │ │ │ ├── basic_layers.py │ │ │ ├── composite_layers.py │ │ │ ├── convolution.py │ │ │ ├── gru.py │ │ │ ├── layers.py │ │ │ ├── lstm.py │ │ │ ├── parser.py │ │ │ ├── stats_layer.py │ │ │ ├── trivial_layers.py │ │ │ └── utils.py │ ├── lmrescore.sh │ ├── lmrescore_const_arpa.sh │ ├── lmrescore_const_arpa_undeterminized.sh │ ├── lmrescore_rnnlm_lat.sh │ ├── make_denlats.sh │ ├── make_denlats_sgmm2.sh │ ├── make_fbank.sh │ ├── make_fbank_pitch.sh │ ├── make_index.sh │ ├── make_mfcc.sh │ ├── make_mfcc_pitch.sh │ ├── make_mfcc_pitch_online.sh │ ├── make_phone_graph.sh │ ├── make_plp.sh │ ├── make_plp_pitch.sh │ ├── nnet │ │ ├── align.sh │ │ ├── decode.sh │ │ ├── ivector │ │ │ ├── extract_ivectors.sh │ │ │ ├── train_diag_ubm.sh │ │ │ └── train_ivector_extractor.sh │ │ ├── make_bn_feats.sh │ │ ├── make_denlats.sh │ │ ├── make_fmllr_feats.sh │ │ ├── make_fmmi_feats.sh │ │ ├── make_priors.sh │ │ ├── pretrain_dbn.sh │ │ ├── train.sh │ │ ├── train_mmi.sh │ │ ├── train_mpe.sh │ │ └── train_scheduler.sh │ ├── nnet2 │ │ ├── adjust_priors.sh │ │ ├── align.sh │ │ ├── check_ivectors_compatible.sh │ │ ├── convert_lda_to_raw.sh │ │ ├── convert_nnet1_to_nnet2.sh │ │ ├── create_appended_model.sh │ │ ├── decode.sh │ │ ├── dump_bottleneck_features.sh │ │ ├── get_egs.sh │ │ ├── get_egs2.sh │ │ ├── get_egs_discriminative2.sh │ │ ├── get_ivector_id.sh │ │ ├── get_lda.sh │ │ ├── get_lda_block.sh │ │ ├── get_num_frames.sh │ │ ├── get_perturbed_feats.sh │ │ ├── make_denlats.sh │ │ ├── make_multisplice_configs.py │ │ ├── relabel_egs.sh │ │ ├── relabel_egs2.sh │ │ ├── remove_egs.sh │ │ ├── retrain_fast.sh │ │ ├── retrain_simple2.sh │ │ ├── retrain_tanh.sh │ │ ├── train_block.sh │ │ ├── train_convnet_accel2.sh │ │ ├── train_discriminative.sh │ │ ├── train_discriminative2.sh │ │ ├── train_discriminative_multilang2.sh │ │ ├── train_more.sh │ │ ├── train_more2.sh │ │ ├── train_multilang2.sh │ │ ├── train_multisplice_accel2.sh │ │ ├── train_multisplice_ensemble.sh │ │ ├── train_pnorm.sh │ │ ├── train_pnorm_accel2.sh │ │ ├── train_pnorm_bottleneck_fast.sh │ │ ├── train_pnorm_ensemble.sh │ │ ├── train_pnorm_fast.sh │ │ ├── train_pnorm_multisplice.sh │ │ ├── train_pnorm_multisplice2.sh │ │ ├── train_pnorm_simple.sh │ │ ├── train_pnorm_simple2.sh │ │ ├── train_tanh.sh │ │ ├── train_tanh_bottleneck.sh │ │ ├── train_tanh_fast.sh │ │ └── update_nnet.sh │ ├── nnet3 │ │ ├── adjust_priors.sh │ │ ├── align.sh │ │ ├── align_lats.sh │ │ ├── chain │ │ │ ├── build_tree.sh │ │ │ ├── build_tree_multiple_sources.sh │ │ │ ├── e2e │ │ │ │ ├── README.txt │ │ │ │ ├── compute_biphone_stats.py │ │ │ │ ├── get_egs_e2e.sh │ │ │ │ ├── prepare_e2e.sh │ │ │ │ ├── text_to_phones.py │ │ │ │ └── train_e2e.py │ │ │ ├── gen_topo.pl │ │ │ ├── gen_topo.py │ │ │ ├── gen_topo2.py │ │ │ ├── gen_topo3.py │ │ │ ├── gen_topo4.py │ │ │ ├── gen_topo5.py │ │ │ ├── gen_topo_orig.py │ │ │ ├── get_egs.sh │ │ │ ├── get_phone_post.sh │ │ │ ├── make_weighted_den_fst.sh │ │ │ ├── multilingual │ │ │ │ └── combine_egs.sh │ │ │ ├── train.py │ │ │ └── train_tdnn.sh │ │ ├── components.py │ │ ├── compute_output.sh │ │ ├── convert_nnet2_to_nnet3.py │ │ ├── decode.sh │ │ ├── decode_grammar.sh │ │ ├── decode_looped.sh │ │ ├── decode_score_fusion.sh │ │ ├── decode_semisup.sh │ │ ├── dot │ │ │ ├── descriptor_parser.py │ │ │ └── nnet3_to_dot.py │ │ ├── get_degs.sh │ │ ├── get_egs.sh │ │ ├── get_egs_discriminative.sh │ │ ├── get_egs_targets.sh │ │ ├── get_saturation.pl │ │ ├── get_successful_models.py │ │ ├── lstm │ │ │ ├── make_configs.py │ │ │ └── train.sh │ │ ├── make_bottleneck_features.sh │ │ ├── make_denlats.sh │ │ ├── make_tdnn_configs.py │ │ ├── multilingual │ │ │ ├── allocate_multilingual_examples.py │ │ │ └── combine_egs.sh │ │ ├── nnet3_to_dot.sh │ │ ├── remove_egs.sh │ │ ├── report │ │ │ ├── convert_model.py │ │ │ ├── generate_plots.py │ │ │ └── summarize_compute_debug_timing.py │ │ ├── tdnn │ │ │ ├── make_configs.py │ │ │ ├── train.sh │ │ │ └── train_raw_nnet.sh │ │ ├── train_discriminative.sh │ │ ├── train_dnn.py │ │ ├── train_raw_dnn.py │ │ ├── train_raw_rnn.py │ │ ├── train_rnn.py │ │ ├── train_tdnn.sh │ │ ├── xconfig_to_config.py │ │ └── xconfig_to_configs.py │ ├── online │ │ ├── decode.sh │ │ ├── nnet2 │ │ │ ├── align.sh │ │ │ ├── copy_data_dir.sh │ │ │ ├── copy_ivector_dir.sh │ │ │ ├── decode.sh │ │ │ ├── dump_nnet_activations.sh │ │ │ ├── extract_ivectors.sh │ │ │ ├── extract_ivectors_online.sh │ │ │ ├── get_egs.sh │ │ │ ├── get_egs2.sh │ │ │ ├── get_egs_discriminative2.sh │ │ │ ├── get_pca_transform.sh │ │ │ ├── make_denlats.sh │ │ │ ├── prepare_online_decoding.sh │ │ │ ├── prepare_online_decoding_retrain.sh │ │ │ ├── prepare_online_decoding_transfer.sh │ │ │ ├── train_diag_ubm.sh │ │ │ └── train_ivector_extractor.sh │ │ ├── nnet3 │ │ │ ├── decode.sh │ │ │ └── prepare_online_decoding.sh │ │ └── prepare_online_decoding.sh │ ├── oracle_wer.sh │ ├── paste_feats.sh │ ├── resegment_data.sh │ ├── resegment_text.sh │ ├── rnnlmrescore.sh │ ├── score_kaldi.sh │ ├── score_kaldi_compare.sh │ ├── scoring │ │ ├── score_kaldi_cer.sh │ │ ├── score_kaldi_compare.sh │ │ └── score_kaldi_wer.sh │ ├── search_index.sh │ ├── segmentation │ │ ├── ali_to_targets.sh │ │ ├── combine_targets_dirs.sh │ │ ├── convert_targets_dir_to_whole_recording.sh │ │ ├── convert_utt2spk_and_segments_to_rttm.py │ │ ├── copy_targets_dir.sh │ │ ├── decode_sad.sh │ │ ├── detect_speech_activity.sh │ │ ├── evaluate_segmentation.pl │ │ ├── get_targets_for_out_of_segments.sh │ │ ├── internal │ │ │ ├── arc_info_to_targets.py │ │ │ ├── find_oov_phone.py │ │ │ ├── get_default_targets_for_out_of_segments.py │ │ │ ├── get_transform_probs_mat.py │ │ │ ├── merge_segment_targets_to_recording.py │ │ │ ├── merge_targets.py │ │ │ ├── prepare_sad_graph.py │ │ │ ├── resample_targets.py │ │ │ ├── sad_to_segments.py │ │ │ └── verify_phones_list.py │ │ ├── lats_to_targets.sh │ │ ├── merge_targets_dirs.sh │ │ ├── post_process_sad_to_segments.sh │ │ ├── prepare_targets_gmm.sh │ │ ├── resample_targets_dir.sh │ │ └── validate_targets_dir.sh │ ├── select_feats.sh │ ├── shift_feats.sh │ ├── subset_ali_dir.sh │ ├── tandem │ │ ├── align_fmllr.sh │ │ ├── align_sgmm2.sh │ │ ├── align_si.sh │ │ ├── decode.sh │ │ ├── decode_fmllr.sh │ │ ├── decode_sgmm2.sh │ │ ├── decode_si.sh │ │ ├── make_denlats.sh │ │ ├── make_denlats_sgmm2.sh │ │ ├── mk_aslf_lda_mllt.sh │ │ ├── mk_aslf_sgmm2.sh │ │ ├── train_deltas.sh │ │ ├── train_lda_mllt.sh │ │ ├── train_mllt.sh │ │ ├── train_mmi.sh │ │ ├── train_mmi_sgmm2.sh │ │ ├── train_mono.sh │ │ ├── train_sat.sh │ │ ├── train_sgmm2.sh │ │ └── train_ubm.sh │ ├── tfrnnlm │ │ ├── check_py.py │ │ ├── check_tensorflow_installed.sh │ │ ├── lmrescore_rnnlm_lat.sh │ │ ├── lmrescore_rnnlm_lat_pruned.sh │ │ ├── lstm.py │ │ ├── lstm_fast.py │ │ ├── reader.py │ │ └── vanilla_rnnlm.py │ ├── train_deltas.sh │ ├── train_diag_ubm.sh │ ├── train_lda_mllt.sh │ ├── train_lvtln.sh │ ├── train_map.sh │ ├── train_mmi.sh │ ├── train_mmi_fmmi.sh │ ├── train_mmi_fmmi_indirect.sh │ ├── train_mmi_sgmm2.sh │ ├── train_mono.sh │ ├── train_mpe.sh │ ├── train_nnet.sh │ ├── train_quick.sh │ ├── train_raw_sat.sh │ ├── train_sat.sh │ ├── train_sat_basis.sh │ ├── train_segmenter.sh │ ├── train_sgmm2.sh │ ├── train_sgmm2_group.sh │ ├── train_smbr.sh │ ├── train_ubm.sh │ └── word_align_lattices.sh │ └── utils │ ├── add_disambig.pl │ ├── add_lex_disambig.pl │ ├── analyze_segments.pl │ ├── apply_map.pl │ ├── best_wer.sh │ ├── build_const_arpa_lm.sh │ ├── combine_data.sh │ ├── convert_ctm.pl │ ├── convert_slf.pl │ ├── convert_slf_parallel.sh │ ├── copy_data_dir.sh │ ├── create_data_link.pl │ ├── create_split_dir.pl │ ├── ctm │ ├── convert_ctm.pl │ ├── fix_ctm.sh │ └── resolve_ctm_overlaps.py │ ├── data │ ├── combine_data.sh │ ├── combine_short_segments.sh │ ├── convert_data_dir_to_whole.sh │ ├── copy_data_dir.sh │ ├── extend_segment_times.py │ ├── extract_wav_segments_data_dir.sh │ ├── fix_data_dir.sh │ ├── fix_subsegment_feats.pl │ ├── get_frame_shift.sh │ ├── get_num_frames.sh │ ├── get_reco2dur.sh │ ├── get_reco2utt_for_data.sh │ ├── get_segments_for_data.sh │ ├── get_uniform_subsegments.py │ ├── get_utt2dur.sh │ ├── get_utt2num_frames.sh │ ├── internal │ │ ├── choose_utts_to_combine.py │ │ ├── combine_segments_to_recording.py │ │ ├── modify_speaker_info.py │ │ └── perturb_volume.py │ ├── limit_feature_dim.sh │ ├── modify_speaker_info.sh │ ├── modify_speaker_info_to_recording.sh │ ├── normalize_data_range.pl │ ├── perturb_data_dir_speed.sh │ ├── perturb_data_dir_speed_3way.sh │ ├── perturb_data_dir_volume.sh │ ├── perturb_speed_to_allowed_lengths.py │ ├── remove_dup_utts.sh │ ├── resample_data_dir.sh │ ├── shift_and_combine_feats.sh │ ├── shift_feats.sh │ ├── split_data.sh │ ├── subsegment_data_dir.sh │ ├── subset_data_dir.sh │ └── validate_data_dir.sh │ ├── dict_dir_add_pronprobs.sh │ ├── eps2disambig.pl │ ├── filt.py │ ├── filter_scp.pl │ ├── filter_scps.pl │ ├── find_arpa_oovs.pl │ ├── fix_ctm.sh │ ├── fix_data_dir.sh │ ├── format_lm.sh │ ├── format_lm_sri.sh │ ├── gen_topo.pl │ ├── int2sym.pl │ ├── kwslist_post_process.pl │ ├── lang │ ├── add_lex_disambig.pl │ ├── add_unigrams_arpa.pl │ ├── adjust_unk_arpa.pl │ ├── adjust_unk_graph.sh │ ├── bpe │ │ ├── add_final_optional_silence.sh │ │ ├── apply_bpe.py │ │ ├── bidi.py │ │ ├── learn_bpe.py │ │ ├── prepend_words.py │ │ └── reverse.py │ ├── check_g_properties.pl │ ├── check_phones_compatible.sh │ ├── compute_sentence_probs_arpa.py │ ├── extend_lang.sh │ ├── get_word_position_phone_map.pl │ ├── grammar │ │ ├── augment_phones_txt.py │ │ └── augment_words_txt.py │ ├── internal │ │ ├── apply_unk_lm.sh │ │ ├── arpa2fst_constrained.py │ │ └── modify_unk_pron.py │ ├── limit_arpa_unk_history.py │ ├── make_kn_lm.py │ ├── make_lexicon_fst.py │ ├── make_lexicon_fst_silprob.py │ ├── make_phone_bigram_lang.sh │ ├── make_phone_lm.py │ ├── make_position_dependent_subword_lexicon.py │ ├── make_subword_lexicon_fst.py │ ├── make_unk_lm.sh │ ├── prepare_lang.sh │ ├── validate_disambig_sym_file.pl │ └── validate_lang.pl │ ├── ln.pl │ ├── make_absolute.sh │ ├── make_lexicon_fst.pl │ ├── make_lexicon_fst_silprob.pl │ ├── make_unigram_grammar.pl │ ├── map_arpa_lm.pl │ ├── mkgraph.sh │ ├── nnet-cpu │ ├── make_nnet_config.pl │ ├── make_nnet_config_block.pl │ ├── make_nnet_config_preconditioned.pl │ └── update_learning_rates.pl │ ├── nnet │ ├── gen_dct_mat.py │ ├── gen_hamm_mat.py │ ├── gen_splice.py │ ├── make_blstm_proto.py │ ├── make_cnn_proto.py │ ├── make_lstm_proto.py │ ├── make_nnet_proto.py │ └── subset_data_tr_cv.sh │ ├── parallel │ ├── limit_num_gpus.sh │ ├── pbs.pl │ ├── queue.pl │ ├── retry.pl │ ├── run.pl │ └── slurm.pl │ ├── parse_options.sh │ ├── pbs.pl │ ├── perturb_data_dir_speed.sh │ ├── pinyin_map.pl │ ├── prepare_extended_lang.sh │ ├── prepare_lang.sh │ ├── prepare_online_nnet_dist_build.sh │ ├── queue.pl │ ├── remove_data_links.sh │ ├── remove_oovs.pl │ ├── retry.pl │ ├── reverse_arpa.py │ ├── rnnlm_compute_scores.sh │ ├── run.pl │ ├── s2eps.pl │ ├── scoring │ ├── wer_ops_details.pl │ ├── wer_per_spk_details.pl │ ├── wer_per_utt_details.pl │ └── wer_report.pl │ ├── segmentation.pl │ ├── show_lattice.sh │ ├── shuffle_list.pl │ ├── slurm.pl │ ├── spk2utt_to_utt2spk.pl │ ├── split_data.sh │ ├── split_scp.pl │ ├── ssh.pl │ ├── subset_data_dir.sh │ ├── subset_data_dir_tr_cv.sh │ ├── subset_scp.pl │ ├── subword │ ├── prepare_lang_subword.sh │ └── prepare_subword_text.sh │ ├── summarize_logs.pl │ ├── summarize_warnings.pl │ ├── sym2int.pl │ ├── utt2spk_to_spk2utt.pl │ ├── validate_data_dir.sh │ ├── validate_dict_dir.pl │ ├── validate_lang.pl │ ├── validate_text.pl │ └── write_kwslist.pl ├── espnet ├── __init__.py ├── asr │ ├── __init__.py │ ├── asr_mix_utils.py │ ├── asr_utils.py │ ├── chainer_backend │ │ ├── __init__.py │ │ └── asr.py │ └── pytorch_backend │ │ ├── __init__.py │ │ ├── asr.py │ │ ├── asr_init.py │ │ ├── asr_mix.py │ │ └── recog.py ├── bin │ ├── __init__.py │ ├── asr_enhance.py │ ├── asr_recog.py │ ├── asr_train.py │ ├── lm_train.py │ ├── mt_train.py │ ├── mt_trans.py │ ├── st_train.py │ ├── st_trans.py │ ├── tts_decode.py │ └── tts_train.py ├── lm │ ├── __init__.py │ ├── chainer_backend │ │ ├── __init__.py │ │ ├── extlm.py │ │ └── lm.py │ ├── lm_utils.py │ └── pytorch_backend │ │ ├── __init__.py │ │ ├── extlm.py │ │ └── lm.py ├── mt │ ├── __init__.py │ ├── mt_utils.py │ └── pytorch_backend │ │ ├── __init__.py │ │ └── mt.py ├── nets │ ├── __init__.py │ ├── asr_interface.py │ ├── batch_beam_search.py │ ├── beam_search.py │ ├── chainer_backend │ │ ├── __init__.py │ │ ├── asr_interface.py │ │ ├── ctc.py │ │ ├── deterministic_embed_id.py │ │ ├── e2e_asr.py │ │ ├── e2e_asr_transformer.py │ │ ├── nets_utils.py │ │ ├── rnn │ │ │ ├── __init__.py │ │ │ ├── attentions.py │ │ │ ├── decoders.py │ │ │ ├── encoders.py │ │ │ └── training.py │ │ └── transformer │ │ │ ├── __init__.py │ │ │ ├── attention.py │ │ │ ├── ctc.py │ │ │ ├── decoder.py │ │ │ ├── decoder_layer.py │ │ │ ├── embedding.py │ │ │ ├── encoder.py │ │ │ ├── encoder_layer.py │ │ │ ├── label_smoothing_loss.py │ │ │ ├── layer_norm.py │ │ │ ├── mask.py │ │ │ ├── plot.py │ │ │ ├── positionwise_feed_forward.py │ │ │ ├── subsampling.py │ │ │ └── training.py │ ├── ctc_prefix_score.py │ ├── e2e_asr_common.py │ ├── e2e_mt_common.py │ ├── lm_interface.py │ ├── mt_interface.py │ ├── pytorch_backend │ │ ├── __init__.py │ │ ├── compressive_transducer │ │ │ ├── __init__.py │ │ │ ├── encoder.py │ │ │ ├── encoder_layer.py │ │ │ ├── initializer.py │ │ │ ├── loss.py │ │ │ ├── rnn_decoders.py │ │ │ ├── transformer_decoder.py │ │ │ ├── transformer_decoder_layer.py │ │ │ ├── utils.py │ │ │ └── vgg.py │ │ ├── ctc.py │ │ ├── e2e_asr.py │ │ ├── e2e_asr_compressive_transducer.py │ │ ├── e2e_asr_mix.py │ │ ├── e2e_asr_mulenc.py │ │ ├── e2e_asr_sync_transducer.py │ │ ├── e2e_asr_transducer.py │ │ ├── e2e_asr_transformer.py │ │ ├── e2e_mt.py │ │ ├── e2e_mt_transformer.py │ │ ├── e2e_st.py │ │ ├── e2e_st_transformer.py │ │ ├── e2e_tts_fastspeech.py │ │ ├── e2e_tts_tacotron2.py │ │ ├── e2e_tts_transformer.py │ │ ├── fastspeech │ │ │ ├── __init__.py │ │ │ ├── duration_calculator.py │ │ │ ├── duration_predictor.py │ │ │ └── length_regulator.py │ │ ├── frontends │ │ │ ├── __init__.py │ │ │ ├── beamformer.py │ │ │ ├── dnn_beamformer.py │ │ │ ├── dnn_wpe.py │ │ │ ├── feature_transform.py │ │ │ ├── frontend.py │ │ │ └── mask_estimator.py │ │ ├── initialization.py │ │ ├── lm │ │ │ ├── __init__.py │ │ │ ├── default.py │ │ │ ├── seq_rnn.py │ │ │ └── transformer.py │ │ ├── memory_component.py │ │ ├── nets_utils.py │ │ ├── rnn │ │ │ ├── __init__.py │ │ │ ├── attentions.py │ │ │ ├── decoders.py │ │ │ └── encoders.py │ │ ├── streaming │ │ │ ├── __init__.py │ │ │ ├── segment.py │ │ │ └── window.py │ │ ├── sync_transducer │ │ │ ├── __init__.py │ │ │ ├── encoder.py │ │ │ ├── encoder_layer.py │ │ │ ├── initializer.py │ │ │ ├── loss.py │ │ │ ├── rnn_decoders.py │ │ │ ├── transformer_decoder.py │ │ │ ├── transformer_decoder_layer.py │ │ │ ├── utils.py │ │ │ └── vgg.py │ │ ├── tacotron2 │ │ │ ├── __init__.py │ │ │ ├── cbhg.py │ │ │ ├── decoder.py │ │ │ └── encoder.py │ │ ├── transducer │ │ │ ├── __init__.py │ │ │ ├── initializer.py │ │ │ ├── loss.py │ │ │ ├── rnn_decoders.py │ │ │ ├── transformer_decoder.py │ │ │ ├── transformer_decoder_layer.py │ │ │ ├── utils.py │ │ │ └── vgg.py │ │ ├── transformer │ │ │ ├── __init__.py │ │ │ ├── add_sos_eos.py │ │ │ ├── additional_loss.py │ │ │ ├── attention.py │ │ │ ├── decoder.py │ │ │ ├── decoder_layer.py │ │ │ ├── embedding.py │ │ │ ├── encoder.py │ │ │ ├── encoder_layer.py │ │ │ ├── initializer.py │ │ │ ├── label_smoothing_loss.py │ │ │ ├── layer_norm.py │ │ │ ├── mask.py │ │ │ ├── multi_layer_conv.py │ │ │ ├── optimizer.py │ │ │ ├── plot.py │ │ │ ├── positionwise_feed_forward.py │ │ │ ├── repeat.py │ │ │ └── subsampling.py │ │ └── wavenet.py │ ├── scorer_interface.py │ ├── scorers │ │ ├── __init__.py │ │ ├── ctc.py │ │ └── length_bonus.py │ ├── st_interface.py │ └── tts_interface.py ├── optimizer │ ├── __init__.py │ ├── chainer.py │ ├── factory.py │ ├── parser.py │ └── pytorch.py ├── scheduler │ ├── __init__.py │ ├── chainer.py │ ├── pytorch.py │ └── scheduler.py ├── st │ └── pytorch_backend │ │ ├── __init__.py │ │ └── st.py ├── transform │ ├── __init__.py │ ├── add_deltas.py │ ├── channel_selector.py │ ├── cmvn.py │ ├── functional.py │ ├── perturb.py │ ├── spec_augment.py │ ├── spectrogram.py │ ├── transform_interface.py │ ├── transformation.py │ └── wpe.py ├── tts │ └── pytorch_backend │ │ ├── __init__.py │ │ └── tts.py └── utils │ ├── __init__.py │ ├── check_kwargs.py │ ├── cli_readers.py │ ├── cli_utils.py │ ├── cli_writers.py │ ├── dataset.py │ ├── deterministic_utils.py │ ├── dynamic_import.py │ ├── fill_missing_args.py │ ├── io_utils.py │ ├── spec_augment.py │ └── training │ ├── __init__.py │ ├── batchfy.py │ ├── evaluator.py │ ├── iterators.py │ ├── tensorboard_logger.py │ └── train_utils.py ├── setup.cfg ├── setup.py ├── tools ├── Makefile ├── check_install.py ├── installers │ ├── install_chainer_ctc.sh │ ├── install_kenlm.sh │ ├── install_mwerSegmenter.sh │ ├── install_nkf.sh │ ├── install_pesq.sh │ ├── install_pyopenjtalk.sh │ ├── install_sctk.sh │ ├── install_sph2pipe.sh │ ├── install_warp-ctc.sh │ └── install_warp-transducer.sh └── setup_anaconda.sh └── utils ├── addjson.py ├── apply-cmvn.py ├── average_checkpoints.py ├── change_yaml.py ├── compute-cmvn-stats.py ├── compute-fbank-feats.py ├── compute-stft-feats.py ├── concatjson.py ├── convert_fbank.sh ├── convert_fbank_to_wav.py ├── copy-feats.py ├── data2json.sh ├── download_from_google_drive.sh ├── dump-pcm.py ├── dump.sh ├── dump_pcm.sh ├── eval-source-separation.py ├── eval_perm_free_error.py ├── eval_source_separation.sh ├── feat-to-shape.py ├── feat_to_shape.sh ├── feats2npy.py ├── filt.py ├── free-gpu.sh ├── generate_wav.sh ├── generate_wav_from_fbank.py ├── get_yaml.py ├── json2sctm.py ├── json2text.py ├── json2trn.py ├── json2trn_mt.py ├── json2trn_wo_dict.py ├── make_fbank.sh ├── make_stft.sh ├── merge_scp2json.py ├── mergejson.py ├── mix-mono-wav-scp.py ├── pack_model.sh ├── queue-freegpu.pl ├── recog_wav.sh ├── reduce_data_dir.sh ├── remove_longshortdata.sh ├── result2json.py ├── score_bleu.sh ├── score_sclite.sh ├── score_sclite_wo_dict.sh ├── scp2json.py ├── show_result.sh ├── splitjson.py ├── spm_decode ├── spm_encode ├── spm_train ├── stdout.pl ├── synth_wav.sh ├── text2token.py ├── text2vocabulary.py ├── trim_silence.py ├── trim_silence.sh ├── trn2ctm.py └── trn2stm.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 kobekobe0410853 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Machine_Translation/.github/workflows/build_wheels.yml: -------------------------------------------------------------------------------- 1 | name: build_wheels 2 | 3 | on: 4 | push: 5 | branches: 6 | - v[0-9]+.[0-9]+.[x0-9]+ 7 | tags: 8 | - v* 9 | 10 | jobs: 11 | build_wheels: 12 | name: Build wheels on ${{ matrix.os }} 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | matrix: 16 | os: [ubuntu-latest, macos-latest] 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | 21 | - name: Install Python 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: '3.7' 25 | 26 | - name: Install cibuildwheel 27 | run: | 28 | python -m pip install cibuildwheel 29 | 30 | - name: Build wheels for CPython 31 | run: | 32 | python -m cibuildwheel --output-dir dist 33 | env: 34 | CIBW_BUILD: "cp36-*64 cp37-*64 cp38-*64" 35 | CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 36 | CIBW_BEFORE_BUILD: git submodule update --init --recursive && pip install . 37 | 38 | - uses: actions/upload-artifact@v2 39 | with: 40 | name: wheels 41 | path: ./dist/*.whl 42 | -------------------------------------------------------------------------------- /Machine_Translation/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "fairseq/model_parallel/megatron"] 2 | path = fairseq/model_parallel/megatron 3 | url = https://github.com/ngoyal2707/Megatron-LM 4 | branch = fairseq 5 | -------------------------------------------------------------------------------- /Machine_Translation/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Facebook, Inc. and its affiliates. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Machine_Translation/docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = fairseq 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /Machine_Translation/docs/_static/theme_overrides.css: -------------------------------------------------------------------------------- 1 | .wy-table-responsive table td kbd { 2 | white-space: nowrap; 3 | } 4 | .wy-table-responsive table td { 5 | white-space: normal !important; 6 | } 7 | .wy-table-responsive { 8 | overflow: visible !important; 9 | } 10 | -------------------------------------------------------------------------------- /Machine_Translation/docs/criterions.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | .. _Criterions: 5 | 6 | Criterions 7 | ========== 8 | 9 | Criterions compute the loss function given the model and batch, roughly:: 10 | 11 | loss = criterion(model, batch) 12 | 13 | .. automodule:: fairseq.criterions 14 | :members: 15 | 16 | .. autoclass:: fairseq.criterions.FairseqCriterion 17 | :members: 18 | :undoc-members: 19 | 20 | .. autoclass:: fairseq.criterions.adaptive_loss.AdaptiveLoss 21 | :members: 22 | :undoc-members: 23 | .. autoclass:: fairseq.criterions.composite_loss.CompositeLoss 24 | :members: 25 | :undoc-members: 26 | .. autoclass:: fairseq.criterions.cross_entropy.CrossEntropyCriterion 27 | :members: 28 | :undoc-members: 29 | .. autoclass:: fairseq.criterions.label_smoothed_cross_entropy.LabelSmoothedCrossEntropyCriterion 30 | :members: 31 | :undoc-members: 32 | -------------------------------------------------------------------------------- /Machine_Translation/docs/docutils.conf: -------------------------------------------------------------------------------- 1 | [writers] 2 | option-limit=0 3 | -------------------------------------------------------------------------------- /Machine_Translation/docs/fairseq.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/docs/fairseq.gif -------------------------------------------------------------------------------- /Machine_Translation/docs/fairseq_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/docs/fairseq_logo.png -------------------------------------------------------------------------------- /Machine_Translation/docs/lr_scheduler.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | .. _Learning Rate Schedulers: 5 | 6 | Learning Rate Schedulers 7 | ======================== 8 | 9 | Learning Rate Schedulers update the learning rate over the course of training. 10 | Learning rates can be updated after each update via :func:`step_update` or at 11 | epoch boundaries via :func:`step`. 12 | 13 | .. automodule:: fairseq.optim.lr_scheduler 14 | :members: 15 | 16 | .. autoclass:: fairseq.optim.lr_scheduler.FairseqLRScheduler 17 | :members: 18 | :undoc-members: 19 | 20 | .. autoclass:: fairseq.optim.lr_scheduler.cosine_lr_scheduler.CosineSchedule 21 | :members: 22 | :undoc-members: 23 | .. autoclass:: fairseq.optim.lr_scheduler.fixed_schedule.FixedSchedule 24 | :members: 25 | :undoc-members: 26 | .. autoclass:: fairseq.optim.lr_scheduler.inverse_square_root_schedule.InverseSquareRootSchedule 27 | :members: 28 | :undoc-members: 29 | .. autoclass:: fairseq.optim.lr_scheduler.reduce_lr_on_plateau.ReduceLROnPlateau 30 | :members: 31 | :undoc-members: 32 | .. autoclass:: fairseq.optim.lr_scheduler.triangular_lr_scheduler.TriangularSchedule 33 | :members: 34 | :undoc-members: 35 | -------------------------------------------------------------------------------- /Machine_Translation/docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=python -msphinx 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=fairseq 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The Sphinx module was not found. Make sure you have Sphinx installed, 20 | echo.then set the SPHINXBUILD environment variable to point to the full 21 | echo.path of the 'sphinx-build' executable. Alternatively you may add the 22 | echo.Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /Machine_Translation/docs/modules.rst: -------------------------------------------------------------------------------- 1 | Modules 2 | ======= 3 | 4 | Fairseq provides several stand-alone :class:`torch.nn.Module` classes that may 5 | be helpful when implementing a new :class:`~fairseq.models.BaseFairseqModel`. 6 | 7 | .. automodule:: fairseq.modules 8 | :members: 9 | :undoc-members: 10 | -------------------------------------------------------------------------------- /Machine_Translation/docs/optim.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | .. _optimizers: 5 | 6 | Optimizers 7 | ========== 8 | 9 | Optimizers update the Model parameters based on the gradients. 10 | 11 | .. automodule:: fairseq.optim 12 | :members: 13 | 14 | .. autoclass:: fairseq.optim.FairseqOptimizer 15 | :members: 16 | :undoc-members: 17 | 18 | .. autoclass:: fairseq.optim.adadelta.Adadelta 19 | :members: 20 | :undoc-members: 21 | .. autoclass:: fairseq.optim.adagrad.Adagrad 22 | :members: 23 | :undoc-members: 24 | .. autoclass:: fairseq.optim.adafactor.FairseqAdafactor 25 | :members: 26 | :undoc-members: 27 | .. autoclass:: fairseq.optim.adam.FairseqAdam 28 | :members: 29 | :undoc-members: 30 | .. autoclass:: fairseq.optim.fp16_optimizer.FP16Optimizer 31 | :members: 32 | :undoc-members: 33 | .. autoclass:: fairseq.optim.nag.FairseqNAG 34 | :members: 35 | :undoc-members: 36 | .. autoclass:: fairseq.optim.sgd.SGD 37 | :members: 38 | :undoc-members: 39 | -------------------------------------------------------------------------------- /Machine_Translation/docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx<2.0 2 | sphinx-argparse 3 | -------------------------------------------------------------------------------- /Machine_Translation/examples/.gitignore: -------------------------------------------------------------------------------- 1 | !*/*.sh 2 | !*/*.md 3 | -------------------------------------------------------------------------------- /Machine_Translation/examples/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | try: 7 | from fairseq.version import __version__ # noqa 8 | except ImportError: 9 | pass 10 | -------------------------------------------------------------------------------- /Machine_Translation/examples/adaptive_span/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import importlib 7 | import os 8 | 9 | # automatically import any Python files in the current directory 10 | cur_dir = os.path.dirname(__file__) 11 | for file in os.listdir(cur_dir): 12 | path = os.path.join(cur_dir, file) 13 | if ( 14 | not file.startswith("_") 15 | and not file.startswith(".") 16 | and (file.endswith(".py") or os.path.isdir(path)) 17 | ): 18 | mod_name = file[: file.find(".py")] if file.endswith(".py") else file 19 | module = importlib.import_module(__name__ + "." + mod_name) 20 | -------------------------------------------------------------------------------- /Machine_Translation/examples/adaptive_span/truncated_bptt_lm_task.py: -------------------------------------------------------------------------------- 1 | ../truncated_bptt/truncated_bptt_lm_task.py -------------------------------------------------------------------------------- /Machine_Translation/examples/backtranslation/sacrebleu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -ne 5 ]; then 4 | echo "usage: $0 [dataset=wmt14/full] [langpair=en-de] [databin] [bpecode] [model]" 5 | exit 6 | fi 7 | 8 | 9 | DATASET=$1 10 | LANGPAIR=$2 11 | DATABIN=$3 12 | BPECODE=$4 13 | MODEL=$5 14 | 15 | SRCLANG=$(echo $LANGPAIR | cut -d '-' -f 1) 16 | TGTLANG=$(echo $LANGPAIR | cut -d '-' -f 2) 17 | 18 | 19 | BPEROOT=examples/backtranslation/subword-nmt/subword_nmt 20 | if [ ! -e $BPEROOT ]; then 21 | BPEROOT=subword-nmt/subword_nmt 22 | if [ ! -e $BPEROOT ]; then 23 | echo 'Cloning Subword NMT repository (for BPE pre-processing)...' 24 | git clone https://github.com/rsennrich/subword-nmt.git 25 | fi 26 | fi 27 | 28 | 29 | sacrebleu -t $DATASET -l $LANGPAIR --echo src \ 30 | | sacremoses tokenize -a -l $SRCLANG -q \ 31 | | python $BPEROOT/apply_bpe.py -c $BPECODE \ 32 | | fairseq-interactive $DATABIN --path $MODEL \ 33 | -s $SRCLANG -t $TGTLANG \ 34 | --beam 5 --remove-bpe --buffer-size 1024 --max-tokens 8000 \ 35 | | grep ^H- | cut -f 3- \ 36 | | sacremoses detokenize -l $TGTLANG -q \ 37 | | sacrebleu -t $DATASET -l $LANGPAIR 38 | -------------------------------------------------------------------------------- /Machine_Translation/examples/constrained_decoding/normalize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) Facebook, Inc. and its affiliates. 4 | # 5 | # This source code is licensed under the MIT license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | import sys 9 | 10 | from sacremoses.normalize import MosesPunctNormalizer 11 | 12 | 13 | def main(args): 14 | normalizer = MosesPunctNormalizer(lang=args.lang, penn=args.penn) 15 | for line in sys.stdin: 16 | print(normalizer.normalize(line.rstrip()), flush=True) 17 | 18 | 19 | if __name__ == "__main__": 20 | import argparse 21 | 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument("--lang", "-l", default="en") 24 | parser.add_argument("--penn", "-p", action="store_true") 25 | args = parser.parse_args() 26 | 27 | main(args) 28 | -------------------------------------------------------------------------------- /Machine_Translation/examples/constrained_decoding/tok.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) Facebook, Inc. and its affiliates. 4 | # 5 | # This source code is licensed under the MIT license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | import sys 9 | 10 | import sacremoses 11 | 12 | 13 | def main(args): 14 | """Tokenizes, preserving tabs""" 15 | mt = sacremoses.MosesTokenizer(lang=args.lang) 16 | 17 | def tok(s): 18 | return mt.tokenize(s, return_str=True) 19 | 20 | for line in sys.stdin: 21 | parts = list(map(tok, line.split("\t"))) 22 | print(*parts, sep="\t", flush=True) 23 | 24 | 25 | if __name__ == "__main__": 26 | import argparse 27 | 28 | parser = argparse.ArgumentParser() 29 | parser.add_argument("--lang", "-l", default="en") 30 | parser.add_argument("--penn", "-p", action="store_true") 31 | parser.add_argument("--fields", "-f", help="fields to tokenize") 32 | args = parser.parse_args() 33 | 34 | main(args) 35 | -------------------------------------------------------------------------------- /Machine_Translation/examples/discriminative_reranking_nmt/__init__.py: -------------------------------------------------------------------------------- 1 | from . import criterions, models, tasks # noqa 2 | -------------------------------------------------------------------------------- /Machine_Translation/examples/discriminative_reranking_nmt/criterions/__init__.py: -------------------------------------------------------------------------------- 1 | from .discriminative_reranking_criterion import KLDivergenceRerankingCriterion 2 | 3 | 4 | __all__ = [ 5 | "KLDivergenceRerankingCriterion", 6 | ] 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/discriminative_reranking_nmt/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .discriminative_reranking_model import DiscriminativeNMTReranker 2 | 3 | 4 | __all__ = [ 5 | "DiscriminativeNMTReranker", 6 | ] 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/discriminative_reranking_nmt/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from .discriminative_reranking_task import DiscriminativeRerankingNMTTask 2 | 3 | 4 | __all__ = [ 5 | "DiscriminativeRerankingNMTTask", 6 | ] 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/fast_noisy_channel/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import noisy_channel_translation # noqa 7 | from . import noisy_channel_sequence_generator # noqa 8 | from . import noisy_channel_beam_search # noqa 9 | -------------------------------------------------------------------------------- /Machine_Translation/examples/flores101/flores_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/flores101/flores_logo.png -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/decode/ax_sweep/ngram.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | 3 | common_eval: 4 | results_path: ${decoding.exp_dir}/decode/${decoding.decoder.name}_ax/${dataset.gen_subset} 5 | 6 | hydra: 7 | sweeper: 8 | ax_config: 9 | max_trials: 60 10 | early_stop: 11 | minimize: true 12 | max_epochs_without_improvement: 10 13 | epsilon: 0.025 14 | experiment: 15 | name: ${dataset.gen_subset} 16 | objective_name: wer 17 | minimize: true 18 | parameter_constraints: null 19 | outcome_constraints: null 20 | status_quo: null 21 | client: 22 | verbose_logging: false 23 | random_seed: null 24 | params: 25 | decoding.decoder.lmweight: 26 | type: range 27 | bounds: [0.0, 8.0] 28 | decoding.decoder.wordscore: 29 | type: range 30 | bounds: [-5.0, 5.0] 31 | decoding.decoder.silweight: 32 | type: range 33 | bounds: [-10.0, 0.0] 34 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/decode/ax_sweep/transformer.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | 3 | common_eval: 4 | results_path: ${decoding.exp_dir}/decode/${decoding.decoder.name}_ax/${dataset.gen_subset} 5 | 6 | hydra: 7 | sweeper: 8 | ax_config: 9 | max_trials: 60 10 | early_stop: 11 | minimize: true 12 | max_epochs_without_improvement: 10 13 | epsilon: 0.025 14 | experiment: 15 | name: ${dataset.gen_subset} 16 | objective_name: wer 17 | minimize: true 18 | parameter_constraints: null 19 | outcome_constraints: null 20 | status_quo: null 21 | client: 22 | verbose_logging: false 23 | random_seed: null 24 | params: 25 | decoding.decoder.lmweight: 26 | type: range 27 | bounds: [0.0, 4.0] 28 | decoding.decoder.wordscore: 29 | type: range 30 | bounds: [-5.0, 5.0] 31 | decoding.decoder.silweight: 32 | type: range 33 | bounds: [-8.0, 0.0] 34 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/decode/infer_fsqlm.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - model: null 5 | 6 | hydra: 7 | run: 8 | dir: ${common_eval.results_path}/beam${decoding.beam}_th${decoding.beamthreshold}_lmw${decoding.lmweight}_wrd${decoding.wordscore}_sil${decoding.silweight} 9 | sweep: 10 | dir: ${common_eval.results_path} 11 | subdir: beam${decoding.beam}_th${decoding.beamthreshold}_lmw${decoding.lmweight}_wrd${decoding.wordscore}_sil${decoding.silweight} 12 | 13 | task: 14 | _name: hubert_pretraining 15 | single_target: true 16 | fine_tuning: true 17 | data: ??? 18 | normalize: ??? 19 | 20 | decoding: 21 | type: fairseqlm 22 | lexicon: ??? 23 | lmpath: ??? 24 | beamthreshold: 25 25 | beam: 500 26 | lmweight: 2 27 | wordscore: -1 28 | silweight: 0 29 | unique_wer_file: true 30 | common_eval: 31 | results_path: ??? 32 | path: ??? 33 | post_process: letter 34 | dataset: 35 | max_tokens: 1100000 36 | gen_subset: ??? 37 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/decode/infer_kenlm.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - model: null 5 | 6 | hydra: 7 | run: 8 | dir: ${common_eval.results_path}/beam${decoding.beam}_th${decoding.beamthreshold}_lmw${decoding.lmweight}_wrd${decoding.wordscore}_sil${decoding.silweight} 9 | sweep: 10 | dir: ${common_eval.results_path} 11 | subdir: beam${decoding.beam}_th${decoding.beamthreshold}_lmw${decoding.lmweight}_wrd${decoding.wordscore}_sil${decoding.silweight} 12 | 13 | task: 14 | _name: hubert_pretraining 15 | single_target: true 16 | fine_tuning: true 17 | data: ??? 18 | normalize: ??? 19 | 20 | decoding: 21 | type: kenlm 22 | lexicon: ??? 23 | lmpath: ??? 24 | beamthreshold: 100 25 | beam: 500 26 | lmweight: 2 27 | wordscore: -1 28 | silweight: 0 29 | unique_wer_file: true 30 | common_eval: 31 | results_path: ??? 32 | path: ??? 33 | post_process: letter 34 | dataset: 35 | max_tokens: 1100000 36 | gen_subset: ??? 37 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/decode/infer_viterbi.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - model: null 5 | 6 | hydra: 7 | run: 8 | dir: ${common_eval.results_path}/viterbi 9 | sweep: 10 | dir: ${common_eval.results_path} 11 | subdir: viterbi 12 | 13 | task: 14 | _name: hubert_pretraining 15 | single_target: true 16 | fine_tuning: true 17 | data: ??? 18 | normalize: ??? 19 | 20 | decoding: 21 | type: viterbi 22 | unique_wer_file: true 23 | common_eval: 24 | results_path: ??? 25 | path: ??? 26 | post_process: letter 27 | dataset: 28 | max_tokens: 1100000 29 | gen_subset: ??? 30 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/decode/run/submitit_slurm.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | hydra: 3 | launcher: 4 | cpus_per_task: ${distributed_training.distributed_world_size} 5 | gpus_per_node: ${distributed_training.distributed_world_size} 6 | tasks_per_node: ${hydra.launcher.gpus_per_node} 7 | nodes: 1 8 | mem_gb: 200 9 | timeout_min: 4320 10 | max_num_timeout: 50 11 | name: ${hydra.job.config_name} 12 | submitit_folder: ${hydra.sweep.dir}/submitit 13 | 14 | distributed_training: 15 | distributed_world_size: 1 16 | distributed_no_spawn: true 17 | distributed_port: 29761 18 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/decode/run/submitit_slurm_8gpu.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | hydra: 3 | launcher: 4 | cpus_per_task: ${distributed_training.distributed_world_size} 5 | gpus_per_node: ${distributed_training.distributed_world_size} 6 | tasks_per_node: ${hydra.launcher.gpus_per_node} 7 | nodes: 1 8 | mem_gb: 200 9 | timeout_min: 4320 10 | max_num_timeout: 50 11 | name: ${hydra.job.config_name} 12 | submitit_folder: ${hydra.sweep.dir}/submitit 13 | 14 | distributed_training: 15 | distributed_world_size: 8 16 | distributed_no_spawn: true 17 | distributed_port: 29761 18 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/finetune/ckpt/it1.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | 3 | task: 4 | normalize: false 5 | 6 | model: 7 | w2v_path: /checkpoint/wnhsu/w2v/hubert_final/iter1/hubert.km.randcrop.pmw1_0.puw0_0.grpnorm.ml10.mp0_8.untie.mxsz250000.ufreq1.maxtok1400000.MU400k.s1337.ngpu32/checkpoint_last.pt 8 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/finetune/lm/ls_4gram.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | 3 | criterion: 4 | wer_kenlm_model: /checkpoint/abdo/old_checkpoint02/datasets/librispeech/4-gram.bin 5 | wer_lexicon: /checkpoint/abdo/old_checkpoint02/datasets/librispeech/10h/raw/lexicon_ltr.lst 6 | wer_lm_weight: 2.0 7 | wer_word_score: -1.0 8 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/finetune/run/submitit_reg.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | 3 | hydra: 4 | launcher: 5 | cpus_per_task: 8 6 | gpus_per_node: 8 7 | tasks_per_node: ${hydra.launcher.gpus_per_node} 8 | nodes: 1 9 | comment: null 10 | mem_gb: 384 11 | timeout_min: 4320 12 | max_num_timeout: 100 13 | constraint: volta32gb 14 | name: ${hydra.job.config_name}/${hydra.job.override_dirname} 15 | submitit_folder: ${hydra.sweep.dir}/submitit/%j 16 | 17 | distributed_training: 18 | distributed_world_size: 8 19 | distributed_port: 29671 20 | nprocs_per_node: 8 21 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/pretrain/data/iter1.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | 3 | task: 4 | label_dir: ??? 5 | labels: ["km"] 6 | 7 | model: 8 | label_rate: 100 9 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/pretrain/data/iter2.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | 3 | task: 4 | label_dir: ??? 5 | labels: ["km"] 6 | 7 | model: 8 | label_rate: 50 9 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/config/pretrain/run/submitit_reg.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | 3 | hydra: 4 | launcher: 5 | cpus_per_task: 8 6 | gpus_per_node: 8 7 | tasks_per_node: ${hydra.launcher.gpus_per_node} 8 | nodes: 4 9 | comment: null 10 | mem_gb: 384 11 | timeout_min: 4320 12 | max_num_timeout: 100 13 | constraint: volta32gb 14 | name: ${hydra.job.config_name}/${hydra.job.override_dirname} 15 | submitit_folder: ${hydra.sweep.dir}/submitit/%j 16 | 17 | distributed_training: 18 | distributed_world_size: 32 19 | distributed_port: 29671 20 | nprocs_per_node: 8 21 | -------------------------------------------------------------------------------- /Machine_Translation/examples/hubert/update_ckpt.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | 8 | src_ckpt = "/checkpoint/wnhsu/w2v/archived/hubert_base_ls960_it2.pt" 9 | ref_ckpt = "/checkpoint/wnhsu/w2v/hubert_icassp_oss_v3/iter2_km100-400k-grp-L6/oss.km500_p0_1_s334.pmw1_0.puw0_0.grpnorm.ml10.mp0_8.untie.mxsz250000.ufreq1.maxtok1400000.MU100k.s1337.ngpu32/checkpoint_last.pt" 10 | new_ckpt = "/checkpoint/wnhsu/w2v/archived/hubert_base_ls960_it2_updated.pt" 11 | 12 | 13 | def update_state(state): 14 | state["model"]["label_embs_concat"] = state["model"].pop("label_embs") 15 | state["args"].task = "hubert_pretraining" 16 | state["args"].labels = f"['{state['args'].labels}']" 17 | return state 18 | 19 | 20 | src_state = torch.load(src_ckpt) 21 | src_state = update_state(src_state) 22 | torch.save(src_state, new_ckpt) 23 | -------------------------------------------------------------------------------- /Machine_Translation/examples/language_model/prepare-wikitext-103.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh 3 | 4 | URLS=( 5 | "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip" 6 | ) 7 | FILES=( 8 | "wikitext-103-v1.zip" 9 | ) 10 | 11 | for ((i=0;i<${#URLS[@]};++i)); do 12 | file=${FILES[i]} 13 | if [ -f $file ]; then 14 | echo "$file already exists, skipping download" 15 | else 16 | url=${URLS[i]} 17 | wget "$url" 18 | if [ -f $file ]; then 19 | echo "$url successfully downloaded." 20 | else 21 | echo "$url not successfully downloaded." 22 | exit -1 23 | fi 24 | if [ ${file: -4} == ".tgz" ]; then 25 | tar zxvf $file 26 | elif [ ${file: -4} == ".tar" ]; then 27 | tar xvf $file 28 | elif [ ${file: -4} == ".zip" ]; then 29 | unzip $file 30 | fi 31 | fi 32 | done 33 | cd .. 34 | -------------------------------------------------------------------------------- /Machine_Translation/examples/laser/laser_src/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .laser_task import * # noqa 7 | from .laser_lstm import * # noqa 8 | from .laser_transformer import * # noqa 9 | -------------------------------------------------------------------------------- /Machine_Translation/examples/latent_depth/latent_depth_src/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import multilingual_translation_latent_depth # noqa 7 | from .loss import latent_depth # noqa 8 | from .models import latent_multilingual_transformer # noqa 9 | from .modules import latent_layers # noqa 10 | -------------------------------------------------------------------------------- /Machine_Translation/examples/latent_depth/latent_depth_src/loss/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/latent_depth/latent_depth_src/loss/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/examples/latent_depth/latent_depth_src/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/latent_depth/latent_depth_src/models/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/examples/latent_depth/latent_depth_src/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/latent_depth/latent_depth_src/modules/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/examples/linformer/README.md: -------------------------------------------------------------------------------- 1 | # Linformer: Self-Attention with Linear Complexity (Wang et al., 2020) 2 | 3 | This example contains code to train Linformer models as described in our paper 4 | [Linformer: Self-Attention with Linear Complexity](https://arxiv.org/abs/2006.04768). 5 | 6 | ## Training a new Linformer RoBERTa model 7 | 8 | You can mostly follow the [RoBERTa pretraining README](/examples/roberta/README.pretraining.md), 9 | updating your training command with `--user-dir examples/linformer/linformer_src --arch linformer_roberta_base`. 10 | 11 | ## Citation 12 | 13 | If you use our work, please cite: 14 | 15 | ```bibtex 16 | @article{wang2020linformer, 17 | title={Linformer: Self-Attention with Linear Complexity}, 18 | author={Wang, Sinong and Li, Belinda and Khabsa, Madian and Fang, Han and Ma, Hao}, 19 | journal={arXiv preprint arXiv:2006.04768}, 20 | year={2020} 21 | } 22 | ``` 23 | -------------------------------------------------------------------------------- /Machine_Translation/examples/linformer/linformer_src/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .models import linformer_roberta # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/linformer/linformer_src/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/linformer/linformer_src/models/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/examples/linformer/linformer_src/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/linformer/linformer_src/modules/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/examples/m2m_100/tokenizers/README.md: -------------------------------------------------------------------------------- 1 | # M2M-100 Tokenization 2 | 3 | We apply different tokenization strategies for different languages following the existing literature. Here we provide tok.sh a tokenizer that can be used to reproduce our results. 4 | 5 | To reproduce the results, follow these steps: 6 | 7 | ``` 8 | tgt_lang=... 9 | reference_translation=... 10 | cat generation_output | grep -P "^H" | sort -V | cut -f 3- | sh tok.sh $tgt_lang > hyp 11 | cat $reference_translation |sh tok.sh $tgt_lang > ref 12 | sacrebleu -tok 'none' ref < hyp 13 | ``` 14 | 15 | ## Installation 16 | 17 | Tools needed for all the languages except Arabic can be installed by running install_dependencies.sh 18 | If you want to evaluate Arabic models, please follow the instructions provided here: http://alt.qcri.org/tools/arabic-normalizer/ to install 19 | -------------------------------------------------------------------------------- /Machine_Translation/examples/m2m_100/tokenizers/seg_ja.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | SCRIPT=`realpath $0` 7 | KYTEA=`dirname $SCRIPT`/thirdparty/kytea 8 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$KYTEA/lib:/usr/local/lib 9 | export PATH=$PATH:"$KYTEA/bin" 10 | 11 | cat - | tr -d "[:blank:]" | kytea -notags 12 | -------------------------------------------------------------------------------- /Machine_Translation/examples/m2m_100/tokenizers/seg_ko.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | SCRIPT=`realpath $0` 7 | MECAB=`dirname $SCRIPT`/thirdparty/mecab-0.996-ko-0.9.2 8 | 9 | export PATH=$PATH:"$MECAB/bin":"$MECAB/lib" 10 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"$MECAB/lib" 11 | 12 | cat - | mecab -O wakati 13 | -------------------------------------------------------------------------------- /Machine_Translation/examples/m2m_100/tokenizers/thirdparty/.gitignore: -------------------------------------------------------------------------------- 1 | seg_my.py 2 | indic_nlp_library/ 3 | indic_nlp_resources/ 4 | kytea/ 5 | mecab-0.996-ko-0.9.2.tar.gz 6 | mecab-0.996-ko-0.9.2/ 7 | mosesdecoder/ 8 | wat2020.my-en.zip 9 | wat2020.my-en/ 10 | wmt16-scripts/ 11 | mecab-ko-dic-2.1.1-20180720/ 12 | mecab-ko-dic-2.1.1-20180720.tar.gz -------------------------------------------------------------------------------- /Machine_Translation/examples/m2m_100/tokenizers/tokenize_indic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # Use: echo {text} | python tokenize_indic.py {language} 8 | 9 | import sys 10 | 11 | from indicnlp.normalize.indic_normalize import IndicNormalizerFactory 12 | from indicnlp.tokenize.indic_tokenize import trivial_tokenize 13 | 14 | 15 | factory = IndicNormalizerFactory() 16 | normalizer = factory.get_normalizer( 17 | sys.argv[1], remove_nuktas=False, nasals_mode="do_nothing" 18 | ) 19 | 20 | for line in sys.stdin: 21 | normalized_line = normalizer.normalize(line.strip()) 22 | tokenized_line = " ".join(trivial_tokenize(normalized_line, sys.argv[1])) 23 | print(tokenized_line) 24 | -------------------------------------------------------------------------------- /Machine_Translation/examples/m2m_100/tokenizers/tokenize_thai.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import sys 8 | 9 | from pythainlp import word_tokenize 10 | 11 | 12 | for line in sys.stdin: 13 | print(" ".join(word_tokenize(line.strip()))) 14 | -------------------------------------------------------------------------------- /Machine_Translation/examples/m2m_100/tokenizers/tokenize_zh.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | 8 | import fileinput 9 | 10 | import sacrebleu 11 | 12 | 13 | for line in fileinput.input(): 14 | print(sacrebleu.tokenize_zh(line)) 15 | -------------------------------------------------------------------------------- /Machine_Translation/examples/m2m_100/tokenizers/tokenizer_ar.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | # Please follow the instructions here http://alt.qcri.org/tools/arabic-normalizer/ 8 | # to install tools needed for Arabic 9 | 10 | echo "Please install Arabic tools: http://alt.qcri.org/tools/arabic-normalizer/" 11 | echo "Then update environment variables in tokenizer_ar.sh" 12 | exit 1 13 | 14 | SVMTOOL=... 15 | GOMOSESGO=... 16 | QCRI_ARABIC_NORMALIZER=... 17 | 18 | export PERL5LIB="$SVMTOOL/lib":"$GOMOSESGO/bin/MADA-3.2":$PERL5LIB 19 | 20 | 21 | tempfile=$(mktemp) 22 | cat - > $tempfile 23 | 24 | cd $QCRI_ARABIC_NORMALIZER 25 | 26 | bash qcri_normalizer_mada3.2_aramorph1.2.1.sh $tempfile 27 | cat $tempfile.mada_norm-aramorph.europarl_tok 28 | -------------------------------------------------------------------------------- /Machine_Translation/examples/megatron_11b/detok.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 -u 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import argparse 8 | import fileinput 9 | 10 | import sacremoses 11 | 12 | 13 | def main(): 14 | parser = argparse.ArgumentParser(description="") 15 | parser.add_argument("files", nargs="*", help="input files") 16 | args = parser.parse_args() 17 | 18 | detok = sacremoses.MosesDetokenizer() 19 | 20 | for line in fileinput.input(args.files, openhook=fileinput.hook_compressed): 21 | print( 22 | detok.detokenize(line.strip().split(" ")) 23 | .replace(" @", "") 24 | .replace("@ ", "") 25 | .replace(" =", "=") 26 | .replace("= ", "=") 27 | .replace(" – ", "–") 28 | ) 29 | 30 | 31 | if __name__ == "__main__": 32 | main() 33 | -------------------------------------------------------------------------------- /Machine_Translation/examples/multilingual/ML50_langs.txt: -------------------------------------------------------------------------------- 1 | ar_AR 2 | cs_CZ 3 | de_DE 4 | en_XX 5 | es_XX 6 | et_EE 7 | fi_FI 8 | fr_XX 9 | gu_IN 10 | hi_IN 11 | it_IT 12 | ja_XX 13 | kk_KZ 14 | ko_KR 15 | lt_LT 16 | lv_LV 17 | my_MM 18 | ne_NP 19 | nl_XX 20 | ro_RO 21 | ru_RU 22 | si_LK 23 | tr_TR 24 | vi_VN 25 | zh_CN 26 | af_ZA 27 | az_AZ 28 | bn_IN 29 | fa_IR 30 | he_IL 31 | hr_HR 32 | id_ID 33 | ka_GE 34 | km_KH 35 | mk_MK 36 | ml_IN 37 | mn_MN 38 | mr_IN 39 | pl_PL 40 | ps_AF 41 | pt_XX 42 | sv_SE 43 | sw_KE 44 | ta_IN 45 | te_IN 46 | th_TH 47 | tl_XX 48 | uk_UA 49 | ur_PK 50 | xh_ZA 51 | gl_ES 52 | sl_SI -------------------------------------------------------------------------------- /Machine_Translation/examples/multilingual/data_scripts/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Install dependency 3 | ```bash 4 | pip install -r requirement.txt 5 | ``` 6 | 7 | # Download the data set 8 | ```bash 9 | export WORKDIR_ROOT= 10 | 11 | ``` 12 | The downloaded data will be at $WORKDIR_ROOT/ML50 13 | 14 | # preprocess the data 15 | Install SPM [here](https://github.com/google/sentencepiece) 16 | ```bash 17 | export WORKDIR_ROOT= 18 | export SPM_PATH= 19 | ``` 20 | * $WORKDIR_ROOT/ML50/raw: extracted raw data 21 | * $WORKDIR_ROOT/ML50/dedup: dedup data 22 | * $WORKDIR_ROOT/ML50/clean: data with valid and test sentences removed from the dedup data 23 | 24 | 25 | -------------------------------------------------------------------------------- /Machine_Translation/examples/multilingual/data_scripts/download_ML50_v1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | if [ -z $WORKDIR_ROOT ] ; 9 | then 10 | echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." 11 | exit 12 | fi 13 | 14 | # first run download_wmt20.sh; it will install a few useful tools for other scripts 15 | # TODO: need to print out instructions on downloading a few files which requires manually authentication from the websites 16 | bash ./download_wmt20.sh 17 | 18 | python ./download_wmt19_and_before.py 19 | bash ./download_wat19_my.sh 20 | python ./download_ted_and_extract.py 21 | bash ./download_lotus.sh 22 | bash ./download_iitb.sh 23 | bash ./download_af_xh.sh 24 | 25 | 26 | # IWSLT downloading URLs have changed in between; TODO: fix them: 27 | bash ./download_iwslt_and_extract.sh 28 | 29 | # TODO: globalvoices URLs changed; need to be fixed 30 | bash ./download_flores_data.sh 31 | -------------------------------------------------------------------------------- /Machine_Translation/examples/multilingual/data_scripts/download_iitb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | 9 | if [ -z $WORKDIR_ROOT ] ; 10 | then 11 | echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." 12 | exit 13 | fi 14 | 15 | IITB=$WORKDIR_ROOT/IITB 16 | mkdir -p $IITB 17 | pushd $IITB 18 | 19 | wget http://www.cfilt.iitb.ac.in/~moses/iitb_en_hi_parallel/iitb_corpus_download/parallel.tgz 20 | tar -xvzf parallel.tgz 21 | 22 | wget http://www.cfilt.iitb.ac.in/~moses/iitb_en_hi_parallel/iitb_corpus_download/dev_test.tgz 23 | tar -xvzf dev_test.tgz 24 | 25 | DESTDIR=${WORKDIR_ROOT}/ML50/raw/ 26 | 27 | cp parallel/IITB.en-hi.en $DESTDIR/train.hi_IN-en_XX.en_XX 28 | cp parallel/IITB.en-hi.hi $DESTDIR/train.hi_IN-en_XX.hi_IN 29 | 30 | cp dev_test/dev.en $DESTDIR/valid.hi_IN-en_XX.en_XX 31 | cp dev_test/dev.hi $DESTDIR/valid.hi_IN-en_XX.hi_IN 32 | 33 | cp dev_test/test.en $DESTDIR/test.hi_IN-en_XX.en_XX 34 | cp dev_test/test.hi $DESTDIR/test.hi_IN-en_XX.hi_IN 35 | popd -------------------------------------------------------------------------------- /Machine_Translation/examples/multilingual/data_scripts/preprocess_ML50_v1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | if [ -z $WORKDIR_ROOT ] ; 9 | then 10 | echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." 11 | exit 12 | fi 13 | 14 | if [ -z $SPM_PATH ] ; 15 | then 16 | echo "Please install sentence piecence from https://github.com/google/sentencepiece and set SPM_PATH pointing to the installed spm_encode.py. Exitting..." 17 | exit 18 | fi 19 | 20 | ML50=${WORKDIR_ROOT}/ML50 21 | 22 | mkdir -p $ML50/dedup 23 | mkdir -p $ML50/cleaned_dedup 24 | 25 | python ./dedup_all.py --from-folder $ML50/raw --to-folder $ML50/dedup 26 | python ./remove_valid_test_in_train.py --from-folder $ML50/dedup --to-folder $ML50/clean 27 | python ./binarize.py --raw-folder $ML50/clean -------------------------------------------------------------------------------- /Machine_Translation/examples/multilingual/data_scripts/requirement.txt: -------------------------------------------------------------------------------- 1 | wget 2 | pandas -------------------------------------------------------------------------------- /Machine_Translation/examples/multilingual/data_scripts/utils/strip_sgm.sh: -------------------------------------------------------------------------------- 1 | grep "seg id" | sed 's///g' | sed 's/<\/seg>//g' 2 | -------------------------------------------------------------------------------- /Machine_Translation/examples/multilingual/multilingual_fairseq_gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | lang_pairs="en-fr,en-cs,fr-en,cs-en" 9 | path_2_data=$1 # 10 | lang_list=$2 # 11 | model=$3 # 12 | source_lang=cs 13 | target_lang=en 14 | 15 | fairseq-generate "$path_2_data" \ 16 | --path "$model" \ 17 | --task translation_multi_simple_epoch \ 18 | --gen-subset test \ 19 | --source-lang "$source_lang" \ 20 | --target-lang "$target_lang" \ 21 | --sacrebleu --remove-bpe 'sentencepiece'\ 22 | --batch-size 32 \ 23 | --encoder-langtok "src" \ 24 | --decoder-langtok \ 25 | --lang-dict "$lang_list" \ 26 | --lang-pairs "$lang_pairs" 27 | -------------------------------------------------------------------------------- /Machine_Translation/examples/noisychannel/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .rerank_options import * # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/pointer_generator/pointer_generator_src/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import transformer_pg # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/roberta/commonsense_qa/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import commonsense_qa_task # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/roberta/commonsense_qa/download_cqa_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | OUTDIR=data/CommonsenseQA 8 | 9 | mkdir -p $OUTDIR 10 | 11 | wget -O $OUTDIR/train.jsonl https://s3.amazonaws.com/commensenseqa/train_rand_split.jsonl 12 | wget -O $OUTDIR/valid.jsonl https://s3.amazonaws.com/commensenseqa/dev_rand_split.jsonl 13 | wget -O $OUTDIR/test.jsonl https://s3.amazonaws.com/commensenseqa/test_rand_split_no_answers.jsonl 14 | wget -O $OUTDIR/dict.txt https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt 15 | -------------------------------------------------------------------------------- /Machine_Translation/examples/roberta/config/pretraining/base.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | common: 3 | fp16: true 4 | log_format: json 5 | log_interval: 200 6 | 7 | checkpoint: 8 | no_epoch_checkpoints: true 9 | 10 | task: 11 | _name: masked_lm 12 | data: ??? 13 | sample_break_mode: complete 14 | tokens_per_sample: 512 15 | 16 | criterion: masked_lm 17 | 18 | dataset: 19 | batch_size: 16 20 | ignore_unused_valid_subsets: true 21 | 22 | optimizer: 23 | _name: adam 24 | weight_decay: 0.01 25 | adam_betas: (0.9,0.98) 26 | adam_eps: 1e-06 27 | 28 | lr_scheduler: 29 | _name: polynomial_decay 30 | warmup_updates: 10000 31 | 32 | optimization: 33 | clip_norm: 0 34 | lr: [0.0005] 35 | max_update: 125000 36 | update_freq: [16] 37 | 38 | model: 39 | _name: roberta 40 | max_positions: 512 41 | dropout: 0.1 42 | attention_dropout: 0.1 43 | -------------------------------------------------------------------------------- /Machine_Translation/examples/roberta/wsc/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import wsc_criterion # noqa 7 | from . import wsc_task # noqa 8 | -------------------------------------------------------------------------------- /Machine_Translation/examples/rxf/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import rxf_src # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/rxf/rxf_src/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import label_smoothed_cross_entropy_r3f, sentence_prediction_r3f # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/simultaneous_translation/README.md: -------------------------------------------------------------------------------- 1 | # Simultaneous Translation 2 | Examples of simultaneous translation in fairseq 3 | - [English-to-Japanese text-to-text wait-k model](docs/enja-waitk.md) 4 | - [English-to-Germen text-to-text monotonic multihead attention model](docs/ende-mma.md) 5 | - [English-to-Germen speech-to-text simultaneous translation model](../speech_to_text/docs/simulst_mustc_example.md) 6 | -------------------------------------------------------------------------------- /Machine_Translation/examples/simultaneous_translation/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import models # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/simultaneous_translation/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import importlib 7 | import os 8 | 9 | 10 | for file in sorted(os.listdir(os.path.dirname(__file__))): 11 | if file.endswith(".py") and not file.startswith("_"): 12 | model_name = file[: file.find(".py")] 13 | importlib.import_module( 14 | "examples.simultaneous_translation.models." + model_name 15 | ) 16 | -------------------------------------------------------------------------------- /Machine_Translation/examples/simultaneous_translation/modules/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import os 8 | import importlib 9 | from fairseq import registry 10 | 11 | ( 12 | build_monotonic_attention, 13 | register_monotonic_attention, 14 | MONOTONIC_ATTENTION_REGISTRY, 15 | _, 16 | ) = registry.setup_registry("--simul-type") 17 | 18 | for file in sorted(os.listdir(os.path.dirname(__file__))): 19 | if file.endswith(".py") and not file.startswith("_"): 20 | model_name = file[: file.find(".py")] 21 | importlib.import_module( 22 | "examples.simultaneous_translation.modules." + model_name 23 | ) 24 | -------------------------------------------------------------------------------- /Machine_Translation/examples/simultaneous_translation/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import importlib 7 | import os 8 | 9 | 10 | # automatically import any Python files in the criterions/ directory 11 | for file in sorted(os.listdir(os.path.dirname(__file__))): 12 | if file.endswith(".py") and not file.startswith("_"): 13 | module = file[: file.find(".py")] 14 | importlib.import_module("examples.simultaneous_translation.utils." + module) 15 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/__init__.py: -------------------------------------------------------------------------------- 1 | from . import criterions, models, tasks # noqa 2 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/criterions/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os 3 | 4 | 5 | # ASG loss requires flashlight bindings 6 | files_to_skip = set() 7 | try: 8 | import flashlight.lib.sequence.criterion 9 | except ImportError: 10 | files_to_skip.add("ASG_loss.py") 11 | 12 | for file in sorted(os.listdir(os.path.dirname(__file__))): 13 | if file.endswith(".py") and not file.startswith("_") and file not in files_to_skip: 14 | criterion_name = file[: file.find(".py")] 15 | importlib.import_module( 16 | "examples.speech_recognition.criterions." + criterion_name 17 | ) 18 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .asr_dataset import AsrDataset 7 | 8 | 9 | __all__ = [ 10 | "AsrDataset", 11 | ] 12 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/kaldi/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/speech_recognition/kaldi/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/kaldi/config/kaldi_initializer.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | data_dir: ??? 4 | fst_dir: ??? 5 | in_labels: ??? 6 | kaldi_root: ??? 7 | lm_arpa: ??? 8 | blank_symbol: 9 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/models/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os 3 | 4 | 5 | for file in sorted(os.listdir(os.path.dirname(__file__))): 6 | if file.endswith(".py") and not file.startswith("_"): 7 | model_name = file[: file.find(".py")] 8 | importlib.import_module("examples.speech_recognition.models." + model_name) 9 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/new/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/speech_recognition/new/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/new/conf/hydra/sweeper/ax.yaml: -------------------------------------------------------------------------------- 1 | # @package hydra.sweeper 2 | _target_: hydra_plugins.hydra_ax_sweeper.ax_sweeper.AxSweeper 3 | max_batch_size: null 4 | ax_config: 5 | max_trials: 128 6 | early_stop: 7 | minimize: true 8 | max_epochs_without_improvement: 32 9 | epsilon: 1.0e-05 10 | experiment: 11 | name: ${dataset.gen_subset} 12 | objective_name: wer 13 | minimize: true 14 | parameter_constraints: null 15 | outcome_constraints: null 16 | status_quo: null 17 | client: 18 | verbose_logging: false 19 | random_seed: null 20 | params: 21 | decoding.lmweight: 22 | type: range 23 | bounds: [0.0, 5.0] 24 | decoding.wordscore: 25 | type: range 26 | bounds: [-5.0, 5.0] 27 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/new/conf/infer.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - task: null 5 | - model: null 6 | 7 | hydra: 8 | run: 9 | dir: ${common_eval.results_path}/${dataset.gen_subset} 10 | sweep: 11 | dir: ${common_eval.results_path} 12 | subdir: ${dataset.gen_subset} 13 | common_eval: 14 | results_path: null 15 | path: null 16 | post_process: letter 17 | quiet: true 18 | dataset: 19 | max_tokens: 1000000 20 | gen_subset: test 21 | distributed_training: 22 | distributed_world_size: 1 23 | decoding: 24 | beam: 5 25 | type: viterbi 26 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/new/decoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/speech_recognition/new/decoders/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/new/decoders/decoder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (c) Facebook, Inc. and its affiliates. 4 | # 5 | # This source code is licensed under the MIT license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | from typing import Union 9 | 10 | from fairseq.data.dictionary import Dictionary 11 | 12 | from .decoder_config import DecoderConfig, FlashlightDecoderConfig 13 | from .base_decoder import BaseDecoder 14 | 15 | 16 | def Decoder( 17 | cfg: Union[DecoderConfig, FlashlightDecoderConfig], tgt_dict: Dictionary 18 | ) -> BaseDecoder: 19 | 20 | if cfg.type == "viterbi": 21 | from .viterbi_decoder import ViterbiDecoder 22 | 23 | return ViterbiDecoder(tgt_dict) 24 | if cfg.type == "kenlm": 25 | from .flashlight_decoder import KenLMDecoder 26 | 27 | return KenLMDecoder(cfg, tgt_dict) 28 | if cfg.type == "fairseqlm": 29 | from .flashlight_decoder import FairseqLMDecoder 30 | 31 | return FairseqLMDecoder(cfg, tgt_dict) 32 | raise NotImplementedError(f"Invalid decoder name: {cfg.name}") 33 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/new/decoders/viterbi_decoder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (c) Facebook, Inc. and its affiliates. 4 | # 5 | # This source code is licensed under the MIT license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | import torch 9 | 10 | from typing import List, Dict 11 | 12 | from .base_decoder import BaseDecoder 13 | 14 | 15 | class ViterbiDecoder(BaseDecoder): 16 | def decode( 17 | self, 18 | emissions: torch.FloatTensor, 19 | ) -> List[List[Dict[str, torch.LongTensor]]]: 20 | def get_pred(e): 21 | toks = e.argmax(dim=-1).unique_consecutive() 22 | return toks[toks != self.blank] 23 | 24 | return [[{"tokens": get_pred(x), "score": 0}] for x in emissions] 25 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_recognition/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os 3 | 4 | 5 | for file in sorted(os.listdir(os.path.dirname(__file__))): 6 | if file.endswith(".py") and not file.startswith("_"): 7 | task_name = file[: file.find(".py")] 8 | importlib.import_module("examples.speech_recognition.tasks." + task_name) 9 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_text_joint_to_text/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import tasks, criterions, models # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_text_joint_to_text/configs/mustc_noise.list: -------------------------------------------------------------------------------- 1 | "(Applause) NOISE 2 | "(Laughter) VOICE 3 | "(Laughter)" VOICE 4 | (Applause) NOISE 5 | (Applause). NOISE 6 | (Audience) VOICE 7 | (Audio) NOISE 8 | (Beat) NOISE 9 | (Beatboxing) VOICE 10 | (Beep) NOISE 11 | (Beeps) NOISE 12 | (Cheering) VOICE 13 | (Cheers) VOICE 14 | (Claps) NOISE 15 | (Clicking) NOISE 16 | (Clunk) NOISE 17 | (Coughs) NOISE 18 | (Drums) NOISE 19 | (Explosion) NOISE 20 | (Gasps) VOICE 21 | (Guitar) NOISE 22 | (Honk) NOISE 23 | (Laugher) VOICE 24 | (Laughing) VOICE 25 | (Laughs) VOICE 26 | (Laughter) VOICE 27 | (Laughter). VOICE 28 | (Laughter)... VOICE 29 | (Mumbling) VOICE 30 | (Music) NOISE 31 | (Noise) NOISE 32 | (Recording) VOICE 33 | (Ringing) NOISE 34 | (Shouts) VOICE 35 | (Sigh) VOICE 36 | (Sighs) VOICE 37 | (Silence) NOISE 38 | (Singing) VOICE 39 | (Sings) VOICE 40 | (Spanish) VOICE 41 | (Static) NOISE 42 | (Tones) NOISE 43 | (Trumpet) NOISE 44 | (Video) NOISE 45 | (Video): NOISE 46 | (Voice-over) NOISE 47 | (Whistle) NOISE 48 | (Whistling) NOISE 49 | (video): NOISE 50 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_text_joint_to_text/criterions/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import importlib 7 | import os 8 | 9 | 10 | for file in os.listdir(os.path.dirname(__file__)): 11 | if file.endswith(".py") and not file.startswith("_"): 12 | criterion_name = file[: file.find(".py")] 13 | importlib.import_module( 14 | "examples.speech_text_joint_to_text.criterions." + criterion_name 15 | ) 16 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_text_joint_to_text/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import importlib 7 | import os 8 | 9 | for file in os.listdir(os.path.dirname(__file__)): 10 | if file.endswith(".py") and not file.startswith("_"): 11 | model_name = file[: file.find(".py")] 12 | importlib.import_module( 13 | "examples.speech_text_joint_to_text.models." + model_name 14 | ) 15 | -------------------------------------------------------------------------------- /Machine_Translation/examples/speech_text_joint_to_text/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import importlib 7 | import os 8 | 9 | for file in os.listdir(os.path.dirname(__file__)): 10 | if file.endswith(".py") and not file.startswith("_"): 11 | task_name = file[: file.find(".py")] 12 | importlib.import_module("examples.speech_text_joint_to_text.tasks." + task_name) 13 | -------------------------------------------------------------------------------- /Machine_Translation/examples/translation_moe/translation_moe_src/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import translation_moe # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/translation_moe/translation_moe_src/logsumexp_moe.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | 8 | 9 | class LogSumExpMoE(torch.autograd.Function): 10 | """Standard LogSumExp forward pass, but use *posterior* for the backward. 11 | 12 | See `"Mixture Models for Diverse Machine Translation: Tricks of the Trade" 13 | (Shen et al., 2019) `_. 14 | """ 15 | 16 | @staticmethod 17 | def forward(ctx, logp, posterior, dim=-1): 18 | ctx.save_for_backward(posterior) 19 | ctx.dim = dim 20 | return torch.logsumexp(logp, dim=dim) 21 | 22 | @staticmethod 23 | def backward(ctx, grad_output): 24 | (posterior,) = ctx.saved_tensors 25 | grad_logp = grad_output.unsqueeze(ctx.dim) * posterior 26 | return grad_logp, None, None 27 | -------------------------------------------------------------------------------- /Machine_Translation/examples/truncated_bptt/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import transformer_xl_model, truncated_bptt_lm_task # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/examples/unsupervised_quality_estimation/repeat_lines.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import argparse 7 | import sys 8 | 9 | 10 | def _normalize_spaces(line): 11 | return " ".join(line.split()) 12 | 13 | 14 | def main(): 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument("-i", "--input_file", required=True, type=str) 17 | parser.add_argument("-n", "--repeat_times", required=True, type=int) 18 | parser.add_argument("-o", "--output_file", required=False, type=str) 19 | args = parser.parse_args() 20 | stream = open(args.output_file, "w") if args.output_file else sys.stdout 21 | 22 | for line in open(args.input_file): 23 | for _ in range(args.repeat_times): 24 | stream.write(_normalize_spaces(line) + "\n") 25 | 26 | 27 | if __name__ == "__main__": 28 | main() 29 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/wav2vec/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/wav2vec/unsupervised/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/config/generate/viterbi.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | fairseq: 4 | task: 5 | _name: unpaired_audio_text 6 | labels: phn 7 | data: ??? 8 | sort_by_length: false 9 | shuffle: false 10 | text_data: '' 11 | 12 | common_eval: 13 | path: ??? 14 | quiet: true 15 | 16 | dataset: 17 | gen_subset: valid 18 | batch_size: 1 19 | 20 | w2l_decoder: VITERBI 21 | post_process: silence 22 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .extracted_features_dataset import ExtractedFeaturesDataset 7 | from .random_input_dataset import RandomInputDataset 8 | 9 | 10 | __all__ = [ 11 | "ExtractedFeaturesDataset", 12 | "RandomInputDataset", 13 | ] 14 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/cmd.sh: -------------------------------------------------------------------------------- 1 | # you can change cmd.sh depending on what type of queue you are using. 2 | # If you have no queueing system and want to run on a local machine, you 3 | # can change all instances 'queue.pl' to run.pl (but be careful and run 4 | # commands one by one: most recipes will exhaust the memory on your 5 | # machine). queue.pl works with GridEngine (qsub). slurm.pl works 6 | # with slurm. Different queues are configured differently, with different 7 | # queue names and different ways of specifying things like memory; 8 | # to account for these differences you can create and edit the file 9 | # conf/queue.conf to match your queue's configuration. Search for 10 | # conf/queue.conf in http://kaldi-asr.org/doc/queue.html for more information, 11 | # or search for the string 'default_config' in utils/queue.pl or utils/slurm.pl. 12 | 13 | export train_cmd="run.pl --mem 2G" 14 | export decode_cmd="run.pl --mem 4G" 15 | export mkgraph_cmd="run.pl --mem 8G" 16 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_phone.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # decode into phones (and prepare a new data directory for HMM outputs) 4 | 5 | . ./path.sh 6 | 7 | set -eu 8 | 9 | out_dir= # same as in train.sh 10 | dec_lmparam= # LM hyperparameters (e.g., 7.0.0) 11 | dec_exp= 12 | dec_script= 13 | dec_splits="train valid" 14 | dec_data_dir=$out_dir/dec_data # where to write HMM output 15 | 16 | data_dir=${out_dir}/data 17 | 18 | local/decode.sh --nj 40 --graph_name graph \ 19 | --val_sets "$dec_splits" --decode_script $dec_script \ 20 | $out_dir/exp/$dec_exp $data_dir $data_dir/lang_test 21 | 22 | if [ ! -z $dec_lmparam ]; then 23 | for x in $dec_splits; do 24 | mkdir -p $dec_data_dir/$x 25 | cp $data_dir/$x/{feats.scp,cmvn.scp,utt2spk,spk2utt} $dec_data_dir/$x/ 26 | 27 | tra=$out_dir/exp/$dec_exp/decode_${x}/scoring/${dec_lmparam}.tra 28 | cat $tra | utils/int2sym.pl -f 2- $data_dir/lang/words.txt | \ 29 | sed 's:::g' | sed 's:::g' > $dec_data_dir/${x}/text 30 | utils/fix_data_dir.sh $dec_data_dir/${x} 31 | echo "WER on ${x} is" $(compute-wer ark:$data_dir/${x}_gt/text ark:$dec_data_dir/$x/text | cut -d" " -f2-) 32 | done 33 | fi 34 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_word_step2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # prepare a new data directory of HMM word output 4 | 5 | . ./path.sh 6 | 7 | set -eu 8 | 9 | out_dir= # same as in train.sh 10 | dec_lmparam= # LM hyperparameters (e.g., 7.0.0) 11 | 12 | dec_exp=tri3b # what HMM stage to decode (e.g., tri3b) 13 | dec_suffix=word 14 | dec_splits="train valid" 15 | dec_data_dir=$out_dir/dec_data_word # where to write HMM output 16 | 17 | data_dir=$out_dir/data 18 | wrd_data_dir=$out_dir/data_word 19 | 20 | for x in $dec_splits; do 21 | mkdir -p $dec_data_dir/$x 22 | cp $data_dir/$x/{feats.scp,cmvn.scp,utt2spk,spk2utt} $dec_data_dir/$x/ 23 | 24 | tra=$out_dir/exp/$dec_exp/decode${dec_suffix}_${x}/scoring/${dec_lmparam}.tra 25 | cat $tra | utils/int2sym.pl -f 2- $data_dir/lang_word/words.txt | \ 26 | sed 's:::g' | sed 's:::g' > $dec_data_dir/$x/text 27 | utils/fix_data_dir.sh $dec_data_dir/$x 28 | echo "WER on $x is" $(compute-wer ark:$wrd_data_dir/${x}_gt/text ark:$dec_data_dir/$x/text | cut -d" " -f2-) 29 | done 30 | 31 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/copy_aligned_text.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | for idx, line in enumerate(sys.stdin): 4 | print(f"utt{idx:010d} {line}", end='') -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/decode.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -u 4 | 5 | val_sets="dev_other" 6 | graph_name=graph 7 | decode_suffix="" 8 | decode_script="steps/decode_fmllr.sh" 9 | decode_args="" 10 | nj=60 11 | 12 | . ./cmd.sh 13 | . ./path.sh 14 | . parse_options.sh 15 | 16 | set -x 17 | exp_dir=$1 18 | data_root=$2 19 | lang_test=$3 20 | 21 | graph=$exp_dir/$graph_name 22 | 23 | if [ ! -d $graph ]; then 24 | utils/mkgraph.sh $lang_test $exp_dir $graph 25 | fi 26 | 27 | for part in $val_sets; do 28 | dec_dir=$exp_dir/decode${decode_suffix}_${part} 29 | if [ ! -d $dec_dir ]; then 30 | echo "decoding $part for $exp_dir" 31 | $decode_script --nj $nj --cmd "$decode_cmd" $decode_args \ 32 | $graph $data_root/$part $dec_dir & 33 | else 34 | echo "$dec_dir exists. skip" 35 | fi 36 | done 37 | 38 | wait 39 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lang.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sil_prob=0.5 4 | num_sil_states=3 5 | num_nonsil_states=1 6 | 7 | . ./cmd.sh 8 | . ./path.sh 9 | . parse_options.sh 10 | 11 | set -eux 12 | 13 | dict=$1 14 | data_dir=$2 15 | 16 | dict_dir=$data_dir/local/dict 17 | tmplm_dir=$data_dir/local/lang_tmp 18 | lm_dir=$data_dir/lang 19 | 20 | mkdir -p $dict_dir $tmplm_dir $lm_dir 21 | 22 | # prepare dict 23 | echo "SIL" > $dict_dir/silence_phones.txt 24 | echo "SIL" > $dict_dir/optional_silence.txt 25 | awk '{print $1}' $dict > $dict_dir/nonsilence_phones.txt 26 | 27 | echo "SIL SIL" > $dict_dir/lexicon.txt 28 | echo " SIL" >> $dict_dir/lexicon.txt 29 | awk '{print $1" "$1}' $dict >> $dict_dir/lexicon.txt 30 | 31 | echo "SIL" > $dict_dir/extra_questions.txt 32 | awk '{printf $1" "} END {printf "\n"}' $dict >> $dict_dir/extra_questions.txt 33 | 34 | # prepare lang 35 | utils/prepare_lang.sh --sil-prob $sil_prob --position-dependent-phones false \ 36 | --num_sil_states $num_sil_states --num_nonsil_states $num_nonsil_states \ 37 | $dict_dir "" $tmplm_dir $lm_dir 38 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lang_word.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | num_sil_states=3 4 | num_nonsil_states=1 5 | 6 | . ./cmd.sh 7 | . ./path.sh 8 | . parse_options.sh 9 | 10 | set -eux 11 | 12 | dict=$1 13 | data_dir=$2 14 | lexicon=$3 15 | 16 | dict_dir=$data_dir/local/dict_word 17 | tmplm_dir=$data_dir/local/lang_tmp_word 18 | lm_dir=$data_dir/lang_word 19 | 20 | mkdir -p $dict_dir $tmplm_dir $lm_dir 21 | 22 | # prepare dict 23 | echo "SIL" > $dict_dir/silence_phones.txt 24 | echo "SIL" > $dict_dir/optional_silence.txt 25 | awk '{print $1}' $dict > $dict_dir/nonsilence_phones.txt 26 | 27 | (echo "!SIL SIL"; echo " SIL";) | cat - $lexicon > $dict_dir/lexicon.txt 28 | 29 | echo "SIL" > $dict_dir/extra_questions.txt 30 | awk '{printf $1" "} END {printf "\n"}' $dict >> $dict_dir/extra_questions.txt 31 | 32 | # prepare lang 33 | utils/prepare_lang.sh --position-dependent-phones false \ 34 | --num_sil_states $num_sil_states --num_nonsil_states $num_nonsil_states \ 35 | $dict_dir "" $tmplm_dir $lm_dir 36 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lm.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | langdir="" 4 | lmdir="" 5 | 6 | . ./cmd.sh 7 | . ./path.sh 8 | . parse_options.sh 9 | 10 | arpa_lm=$1 11 | data=$2 12 | 13 | if [ -z $langdir ]; then 14 | langdir=$data/lang 15 | fi 16 | if [ -z $lmdir ]; then 17 | lmdir=$data/lang_test 18 | fi 19 | 20 | if [ ! -d $langdir ]; then 21 | echo "$langdir not found. run local/prepare_lang.sh first" && exit 1 22 | fi 23 | 24 | mkdir -p $lmdir 25 | cp -r $langdir/* $lmdir 26 | 27 | if [[ "$arpa_lm" == *.gz ]]; then 28 | gunzip -c $arpa_lm | arpa2fst --disambig-symbol=#0 --read-symbol-table=$lmdir/words.txt - $lmdir/G.fst 29 | else 30 | arpa2fst --disambig-symbol=#0 --read-symbol-table=$lmdir/words.txt $arpa_lm $lmdir/G.fst 31 | fi 32 | fstisstochastic $lmdir/G.fst 33 | utils/validate_lang.pl $lmdir || exit 1 34 | 35 | echo "done preparing lm ($lmdir)" 36 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select_decode.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | split="dev_other" 4 | ref_txt="" # ground truth transcript path 5 | psd_txt="" # pseudo transcript path 6 | get_best_wer=true 7 | dec_name="decode" 8 | graph_name="graph" 9 | kenlm_path=/checkpoint/abaevski/data/speech/libri/librispeech_lm_novox.phnc_o6.bin 10 | 11 | . ./cmd.sh 12 | . ./path.sh 13 | . parse_options.sh 14 | 15 | exp_root=$1 16 | unsup_args="" 17 | if [ $# -ge 2 ]; then 18 | unsup_args=$2 19 | fi 20 | 21 | set -eu 22 | 23 | if [ ! -z $ref_txt ] && $get_best_wer; then 24 | echo "==== WER w.r.t. real transcript (select based on unsupervised metric)" 25 | for x in $exp_root/*/${dec_name}_${split}*; do 26 | lang=$(dirname $x)/$graph_name 27 | 28 | ( 29 | for tra in $x/scoring/*.tra; do 30 | cat $tra | utils/int2sym.pl -f 2- $lang/words.txt | sed 's:::g' | sed 's:::g' > $tra.txt 31 | python local/unsup_select.py $psd_txt $tra.txt --kenlm_path $kenlm_path --gt_tra $ref_txt $unsup_args 32 | done 2>/dev/null | grep "score=" | sed 's/=/ /g' | sed 's/;//g' | sort -k3n | head -n1 33 | ) & 34 | done 35 | fi 36 | wait 37 | 38 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select_decode_word.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | split="dev_other" 4 | ref_txt="" # ground truth transcript path 5 | psd_txt="" # pseudo transcript path 6 | get_best_wer=true 7 | dec_name="decode" 8 | graph_name="graph" 9 | kenlm_path=/checkpoint/abaevski/data/speech/libri/librispeech_lm_novox.phnc_o6.bin 10 | phonemize_lexicon="" 11 | 12 | . ./cmd.sh 13 | . ./path.sh 14 | . parse_options.sh 15 | . /private/home/wnhsu/unsup_asr/fairseq-py-unsup/env.sh 16 | 17 | exp_root=$1 18 | 19 | set -eu 20 | 21 | if [ ! -z $ref_txt ] && $get_best_wer; then 22 | echo "==== WER w.r.t. real transcript (select based on unsupervised metric)" 23 | for x in $exp_root/*/${dec_name}_${split}*; do 24 | lang=$(dirname $x)/$graph_name 25 | 26 | for tra in $x/scoring/*.tra; do 27 | cat $tra | utils/int2sym.pl -f 2- $lang/words.txt | sed 's:\::g' > $tra.txt 28 | python local/unsup_select.py $psd_txt $tra.txt \ 29 | --kenlm_path $kenlm_path --gt_tra $ref_txt --phonemize \ 30 | --phonemize_lexicon "$phonemize_lexicon" 31 | done | grep "score=" | sed 's/=/ /g' | sed 's/;//g' | sort -k3n | head -n1 32 | done 33 | fi 34 | 35 | 36 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/path.sh: -------------------------------------------------------------------------------- 1 | export KALDI_ROOT=`pwd`/../../.. 2 | export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PWD:$PATH 3 | [ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1 4 | . $KALDI_ROOT/tools/config/common_path.sh 5 | export LC_ALL=C 6 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/steps: -------------------------------------------------------------------------------- 1 | ../../wsj/s5/steps -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/utils: -------------------------------------------------------------------------------- 1 | ../../wsj/s5/utils -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .wav2vec_u import Wav2vec_U 7 | 8 | 9 | __all__ = [ 10 | "Wav2vec_U", 11 | ] 12 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/scripts/copy_labels.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 -u 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import sys 8 | 9 | for idx, line in enumerate(sys.stdin): 10 | print(f"utt{idx:010d} {line}", end="") 11 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/scripts/filter_lexicon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 -u 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import argparse 8 | import sys 9 | 10 | from fairseq.data import Dictionary 11 | 12 | 13 | def get_parser(): 14 | parser = argparse.ArgumentParser( 15 | description="filters a lexicon given a unit dictionary" 16 | ) 17 | parser.add_argument("-d", "--unit-dict", help="unit dictionary", required=True) 18 | return parser 19 | 20 | 21 | def main(): 22 | parser = get_parser() 23 | args = parser.parse_args() 24 | 25 | d = Dictionary.load(args.unit_dict) 26 | symbols = set(d.symbols) 27 | 28 | for line in sys.stdin: 29 | items = line.rstrip().split() 30 | skip = len(items) < 2 31 | for x in items[1:]: 32 | if x not in symbols: 33 | skip = True 34 | break 35 | if not skip: 36 | print(line, end="") 37 | 38 | 39 | if __name__ == "__main__": 40 | main() 41 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/scripts/filter_tsv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 -u 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import argparse 9 | import sys 10 | 11 | 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument("--tsv", required=True, type=str) 14 | parser.add_argument("--no-skip", action="store_true") 15 | parser.add_argument("--keep", action="store_true") 16 | params = parser.parse_args() 17 | 18 | 19 | def get_fname(line): 20 | p = os.path.basename(line.split("\t")[0]) 21 | p = os.path.splitext(p)[0] 22 | return p 23 | 24 | 25 | # filenames to exclude 26 | seen = set() 27 | with open(params.tsv) as f: 28 | if not params.no_skip: 29 | root = next(f).rstrip() 30 | for line in f: 31 | seen.add(get_fname(line)) 32 | 33 | for i, line in enumerate(sys.stdin): 34 | exists = get_fname(line) in seen 35 | keep = (exists and params.keep) or (not exists and not params.keep) 36 | if i == 0 or keep: 37 | print(line, end="") 38 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/scripts/ltr_to_wrd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 -u 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import sys 8 | 9 | 10 | def main(): 11 | for line in sys.stdin: 12 | print(line.replace(" ", "").replace("|", " ").strip()) 13 | 14 | 15 | if __name__ == "__main__": 16 | main() 17 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/scripts/normalize_text.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import regex 8 | import sys 9 | 10 | 11 | def main(): 12 | filter_r = regex.compile(r"[^\p{L}\p{N}\p{M}\' \-]") 13 | 14 | for line in sys.stdin: 15 | line = line.strip() 16 | line = filter_r.sub(" ", line) 17 | line = " ".join(line.split()) 18 | print(line) 19 | 20 | 21 | if __name__ == "__main__": 22 | main() 23 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/scripts/wrd_to_ltr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 -u 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import sys 8 | 9 | 10 | def main(): 11 | for line in sys.stdin: 12 | print(" ".join(list(line.strip().replace(" ", "|"))) + " |") 13 | 14 | 15 | if __name__ == "__main__": 16 | main() 17 | -------------------------------------------------------------------------------- /Machine_Translation/examples/wav2vec/unsupervised/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .unpaired_audio_text import UnpairedAudioText 7 | 8 | 9 | __all__ = [ 10 | "UnpairedAudioText", 11 | ] 12 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # import models/tasks to register them 7 | from . import dummy_dataset, dummy_lm, dummy_masked_lm, dummy_model, dummy_mt # noqa 8 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/benchmark/dummy_dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from fairseq.data import FairseqDataset 3 | 4 | 5 | class DummyDataset(FairseqDataset): 6 | def __init__(self, batch, num_items, item_size): 7 | super().__init__() 8 | self.batch = batch 9 | self.num_items = num_items 10 | self.item_size = item_size 11 | 12 | def __getitem__(self, index): 13 | return index 14 | 15 | def __len__(self): 16 | return self.num_items 17 | 18 | def collater(self, samples): 19 | return self.batch 20 | 21 | @property 22 | def sizes(self): 23 | return np.array([self.item_size] * self.num_items) 24 | 25 | def num_tokens(self, index): 26 | return self.item_size 27 | 28 | def size(self, index): 29 | return self.item_size 30 | 31 | def ordered_indices(self): 32 | return np.arange(self.num_items) 33 | 34 | @property 35 | def supports_prefetch(self): 36 | return False 37 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/clib/libbleu/module.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-present, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the license found in the 6 | * LICENSE file in the root directory of this source tree. 7 | */ 8 | 9 | #include 10 | 11 | static PyMethodDef method_def[] = {{NULL, NULL, 0, NULL}}; // NOLINT 12 | 13 | static struct PyModuleDef module_def = { 14 | PyModuleDef_HEAD_INIT, 15 | "libbleu", /* name of module */ 16 | // NOLINTNEXTLINE 17 | NULL, /* module documentation, may be NULL */ 18 | -1, /* size of per-interpreter state of the module, 19 | or -1 if the module keeps state in global variables. */ 20 | method_def}; // NOLINT 21 | 22 | #if PY_MAJOR_VERSION == 2 23 | PyMODINIT_FUNC init_libbleu() 24 | #else 25 | PyMODINIT_FUNC PyInit_libbleu() 26 | #endif 27 | { 28 | PyObject* m = PyModule_Create(&module_def); 29 | if (!m) { 30 | return NULL; 31 | } 32 | return m; 33 | } 34 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/clib/libnat_cuda/edit_dist.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-present, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the license found in the 6 | * LICENSE file in the root directory of this source tree. 7 | */ 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | torch::Tensor LevenshteinDistanceCuda( 14 | torch::Tensor source, 15 | torch::Tensor target, 16 | torch::Tensor source_length, 17 | torch::Tensor target_length); 18 | 19 | torch::Tensor GenerateDeletionLabelCuda( 20 | torch::Tensor source, 21 | torch::Tensor operations); 22 | 23 | std::pair GenerateInsertionLabelCuda( 24 | torch::Tensor source, 25 | torch::Tensor operations); 26 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/config.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | hydra: 4 | run: 5 | dir: . 6 | 7 | defaults: 8 | - _self_ 9 | - task: null 10 | - model: null 11 | - criterion: cross_entropy 12 | - optimizer: null 13 | - lr_scheduler: fixed 14 | - bpe: null 15 | - tokenizer: null 16 | - scoring: null 17 | - generation: null 18 | - common_eval: null 19 | - eval_lm: null 20 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_baevski_gbw.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | activation_fn: "relu" 3 | dropout: 0.1 4 | attention_dropout: 0.1 5 | activation_dropout: 0.0 6 | relu_dropout: 0.0 7 | decoder_embed_dim: 512 8 | decoder_output_dim: 512 9 | decoder_input_dim: 512 10 | decoder_ffn_embed_dim: 4096 11 | decoder_layers: 12 12 | decoder_attention_heads: 16 13 | decoder_normalize_before: true 14 | no_decoder_final_norm: true 15 | adaptive_softmax_cutoff: null 16 | adaptive_softmax_dropout: 0 17 | adaptive_softmax_factor: 4 18 | no_token_positional_embeddings: false 19 | share_decoder_input_output_embed: false 20 | character_embeddings: false 21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" 22 | character_embedding_dim: 4 23 | char_embedder_highway_layers: 2 24 | adaptive_input: false 25 | adaptive_input_factor: 4 26 | adaptive_input_cutoff: null 27 | tie_adaptive_weights: false 28 | tie_adaptive_proj: false 29 | decoder_learned_pos: false 30 | decoder_layerdrop: 0 31 | decoder_layers_to_keep: null 32 | layernorm_embedding: false 33 | no_scale_embedding: false 34 | quant_noise_pq: 0 35 | quant_noise_pq_block_size: 8 36 | quant_noise_scalar: 0 37 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_big.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | activation_fn: "relu" 3 | dropout: 0.1 4 | attention_dropout: 0.0 5 | activation_dropout: 0.0 6 | relu_dropout: 0.0 7 | decoder_embed_dim: 1024 8 | decoder_output_dim: 1024 9 | decoder_input_dim: 1024 10 | decoder_ffn_embed_dim: 4096 11 | decoder_layers: 12 12 | decoder_attention_heads: 16 13 | decoder_normalize_before: true 14 | no_decoder_final_norm: false 15 | adaptive_softmax_cutoff: null 16 | adaptive_softmax_dropout: 0 17 | adaptive_softmax_factor: 4 18 | no_token_positional_embeddings: false 19 | share_decoder_input_output_embed: false 20 | character_embeddings: false 21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" 22 | character_embedding_dim: 4 23 | char_embedder_highway_layers: 2 24 | adaptive_input: false 25 | adaptive_input_factor: 4 26 | adaptive_input_cutoff: null 27 | tie_adaptive_weights: false 28 | tie_adaptive_proj: false 29 | decoder_learned_pos: false 30 | decoder_layerdrop: 0 31 | decoder_layers_to_keep: null 32 | layernorm_embedding: false 33 | no_scale_embedding: false 34 | quant_noise_pq: 0 35 | quant_noise_pq_block_size: 8 36 | quant_noise_scalar: 0 37 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_gbw.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | activation_fn: "relu" 3 | dropout: 0.1 4 | attention_dropout: 0.1 5 | activation_dropout: 0.0 6 | relu_dropout: 0.0 7 | decoder_embed_dim: 512 8 | decoder_output_dim: 512 9 | decoder_input_dim: 512 10 | decoder_ffn_embed_dim: 4096 11 | decoder_layers: 12 12 | decoder_attention_heads: 16 13 | decoder_normalize_before: true 14 | no_decoder_final_norm: true 15 | adaptive_softmax_cutoff: null 16 | adaptive_softmax_dropout: 0 17 | adaptive_softmax_factor: 4 18 | no_token_positional_embeddings: false 19 | share_decoder_input_output_embed: false 20 | character_embeddings: false 21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" 22 | character_embedding_dim: 4 23 | char_embedder_highway_layers: 2 24 | adaptive_input: false 25 | adaptive_input_factor: 4 26 | adaptive_input_cutoff: null 27 | tie_adaptive_weights: false 28 | tie_adaptive_proj: false 29 | decoder_learned_pos: false 30 | decoder_layerdrop: 0 31 | decoder_layers_to_keep: null 32 | layernorm_embedding: false 33 | no_scale_embedding: false 34 | quant_noise_pq: 0 35 | quant_noise_pq_block_size: 8 36 | quant_noise_scalar: 0 37 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_gpt.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | activation_fn: "gelu" 3 | dropout: 0.1 4 | attention_dropout: 0.1 5 | activation_dropout: 0.0 6 | relu_dropout: 0.0 7 | decoder_embed_dim: 768 8 | decoder_output_dim: 768 9 | decoder_input_dim: 768 10 | decoder_ffn_embed_dim: 3072 11 | decoder_layers: 12 12 | decoder_attention_heads: 12 13 | decoder_normalize_before: true 14 | no_decoder_final_norm: false 15 | adaptive_softmax_cutoff: null 16 | adaptive_softmax_dropout: 0 17 | adaptive_softmax_factor: 4 18 | no_token_positional_embeddings: false 19 | share_decoder_input_output_embed: false 20 | character_embeddings: false 21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" 22 | character_embedding_dim: 4 23 | char_embedder_highway_layers: 2 24 | adaptive_input: false 25 | adaptive_input_factor: 4 26 | adaptive_input_cutoff: null 27 | tie_adaptive_weights: false 28 | tie_adaptive_proj: false 29 | decoder_learned_pos: false 30 | decoder_layerdrop: 0 31 | decoder_layers_to_keep: null 32 | layernorm_embedding: false 33 | no_scale_embedding: false 34 | quant_noise_pq: 0 35 | quant_noise_pq_block_size: 8 36 | quant_noise_scalar: 0 37 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_gpt2_big.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | activation_fn: "gelu" 3 | dropout: 0.1 4 | attention_dropout: 0.1 5 | activation_dropout: 0.0 6 | relu_dropout: 0.0 7 | decoder_embed_dim: 1600 8 | decoder_output_dim: 1600 9 | decoder_input_dim: 1600 10 | decoder_ffn_embed_dim: 6400 11 | decoder_layers: 48 12 | decoder_attention_heads: 25 13 | decoder_normalize_before: true 14 | no_decoder_final_norm: false 15 | adaptive_softmax_cutoff: null 16 | adaptive_softmax_dropout: 0 17 | adaptive_softmax_factor: 4 18 | no_token_positional_embeddings: false 19 | share_decoder_input_output_embed: false 20 | character_embeddings: false 21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" 22 | character_embedding_dim: 4 23 | char_embedder_highway_layers: 2 24 | adaptive_input: false 25 | adaptive_input_factor: 4 26 | adaptive_input_cutoff: null 27 | tie_adaptive_weights: false 28 | tie_adaptive_proj: false 29 | decoder_learned_pos: false 30 | decoder_layerdrop: 0 31 | decoder_layers_to_keep: null 32 | layernorm_embedding: false 33 | no_scale_embedding: false 34 | quant_noise_pq: 0 35 | quant_noise_pq_block_size: 8 36 | quant_noise_scalar: 0 37 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_gpt2_medium.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | activation_fn: "gelu" 3 | dropout: 0.1 4 | attention_dropout: 0.1 5 | activation_dropout: 0.0 6 | relu_dropout: 0.0 7 | decoder_embed_dim: 1280 8 | decoder_output_dim: 1280 9 | decoder_input_dim: 1280 10 | decoder_ffn_embed_dim: 5120 11 | decoder_layers: 36 12 | decoder_attention_heads: 20 13 | decoder_normalize_before: true 14 | no_decoder_final_norm: false 15 | adaptive_softmax_cutoff: null 16 | adaptive_softmax_dropout: 0 17 | adaptive_softmax_factor: 4 18 | no_token_positional_embeddings: false 19 | share_decoder_input_output_embed: false 20 | character_embeddings: false 21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" 22 | character_embedding_dim: 4 23 | char_embedder_highway_layers: 2 24 | adaptive_input: false 25 | adaptive_input_factor: 4 26 | adaptive_input_cutoff: null 27 | tie_adaptive_weights: false 28 | tie_adaptive_proj: false 29 | decoder_learned_pos: false 30 | decoder_layerdrop: 0 31 | decoder_layers_to_keep: null 32 | layernorm_embedding: false 33 | no_scale_embedding: false 34 | quant_noise_pq: 0 35 | quant_noise_pq_block_size: 8 36 | quant_noise_scalar: 0 37 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_gpt2_small.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | activation_fn: "gelu" 3 | dropout: 0.1 4 | attention_dropout: 0.1 5 | activation_dropout: 0.0 6 | relu_dropout: 0.0 7 | decoder_embed_dim: 1024 8 | decoder_output_dim: 1024 9 | decoder_input_dim: 1024 10 | decoder_ffn_embed_dim: 4096 11 | decoder_layers: 24 12 | decoder_attention_heads: 16 13 | decoder_normalize_before: true 14 | no_decoder_final_norm: false 15 | adaptive_softmax_cutoff: null 16 | adaptive_softmax_dropout: 0 17 | adaptive_softmax_factor: 4 18 | no_token_positional_embeddings: false 19 | share_decoder_input_output_embed: false 20 | character_embeddings: false 21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" 22 | character_embedding_dim: 4 23 | char_embedder_highway_layers: 2 24 | adaptive_input: false 25 | adaptive_input_factor: 4 26 | adaptive_input_cutoff: null 27 | tie_adaptive_weights: false 28 | tie_adaptive_proj: false 29 | decoder_learned_pos: false 30 | decoder_layerdrop: 0 31 | decoder_layers_to_keep: null 32 | layernorm_embedding: false 33 | no_scale_embedding: false 34 | quant_noise_pq: 0 35 | quant_noise_pq_block_size: 8 36 | quant_noise_scalar: 0 37 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/model/wav2vec/vq_wav2vec_gumbel.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | activation: gelu 3 | vq_type: gumbel 4 | vq_depth: 2 5 | combine_groups: true 6 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/model/wav2vec2/wav2vec2_base.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | quantize_targets: true 4 | final_dim: 256 5 | encoder_layerdrop: 0.05 6 | dropout_input: 0.1 7 | dropout_features: 0.1 8 | feature_grad_mult: 0.1 9 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/config/model/wav2vec2/wav2vec2_large.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | quantize_targets: true 4 | extractor_mode: layer_norm 5 | layer_norm_first: true 6 | final_dim: 768 7 | latent_temp: [2.0,0.1,0.999995] 8 | encoder_layerdrop: 0.0 9 | dropout_input: 0.0 10 | dropout_features: 0.0 11 | dropout: 0.0 12 | attention_dropout: 0.0 13 | conv_bias: true 14 | 15 | encoder_layers: 24 16 | encoder_embed_dim: 1024 17 | encoder_ffn_embed_dim: 4096 18 | encoder_attention_heads: 16 19 | 20 | feature_grad_mult: 1.0 21 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/criterions/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | """isort:skip_file""" 6 | 7 | import importlib 8 | import os 9 | 10 | from fairseq import registry 11 | from fairseq.criterions.fairseq_criterion import ( # noqa 12 | FairseqCriterion, 13 | LegacyFairseqCriterion, 14 | ) 15 | from omegaconf import DictConfig 16 | 17 | 18 | ( 19 | build_criterion_, 20 | register_criterion, 21 | CRITERION_REGISTRY, 22 | CRITERION_DATACLASS_REGISTRY, 23 | ) = registry.setup_registry( 24 | "--criterion", base_class=FairseqCriterion, default="cross_entropy" 25 | ) 26 | 27 | 28 | def build_criterion(cfg: DictConfig, task): 29 | return build_criterion_(cfg, task) 30 | 31 | 32 | # automatically import any Python files in the criterions/ directory 33 | for file in sorted(os.listdir(os.path.dirname(__file__))): 34 | if file.endswith(".py") and not file.startswith("_"): 35 | file_name = file[: file.find(".py")] 36 | importlib.import_module("fairseq.criterions." + file_name) 37 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/audio/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/fairseq/data/audio/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/audio/feature_transforms/global_cmvn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from fairseq.data.audio.feature_transforms import ( 3 | AudioFeatureTransform, 4 | register_audio_feature_transform, 5 | ) 6 | 7 | 8 | @register_audio_feature_transform("global_cmvn") 9 | class GlobalCMVN(AudioFeatureTransform): 10 | """Global CMVN (cepstral mean and variance normalization). The global mean 11 | and variance need to be pre-computed and stored in NumPy format (.npz).""" 12 | 13 | @classmethod 14 | def from_config_dict(cls, config=None): 15 | _config = {} if config is None else config 16 | return GlobalCMVN(_config.get("stats_npz_path")) 17 | 18 | def __init__(self, stats_npz_path): 19 | self.stats_npz_path = stats_npz_path 20 | stats = np.load(stats_npz_path) 21 | self.mean, self.std = stats["mean"], stats["std"] 22 | 23 | def __repr__(self): 24 | return self.__class__.__name__ + f'(stats_npz_path="{self.stats_npz_path}")' 25 | 26 | def __call__(self, x): 27 | x = np.subtract(x, self.mean) 28 | x = np.divide(x, self.std) 29 | return x 30 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/colorize_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | 8 | from . import BaseWrapperDataset 9 | 10 | 11 | class ColorizeDataset(BaseWrapperDataset): 12 | """ Adds 'colors' property to net input that is obtained from the provided color getter for use by models """ 13 | 14 | def __init__(self, dataset, color_getter): 15 | super().__init__(dataset) 16 | self.color_getter = color_getter 17 | 18 | def collater(self, samples): 19 | base_collate = super().collater(samples) 20 | if len(base_collate) > 0: 21 | base_collate["net_input"]["colors"] = torch.tensor( 22 | list(self.color_getter(self.dataset, s["id"]) for s in samples), 23 | dtype=torch.long, 24 | ) 25 | return base_collate 26 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/encoders/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import importlib 8 | import os 9 | 10 | from fairseq import registry 11 | 12 | 13 | build_tokenizer, register_tokenizer, TOKENIZER_REGISTRY, _ = registry.setup_registry( 14 | "--tokenizer", 15 | default=None, 16 | ) 17 | 18 | 19 | build_bpe, register_bpe, BPE_REGISTRY, _ = registry.setup_registry( 20 | "--bpe", 21 | default=None, 22 | ) 23 | 24 | 25 | # automatically import any Python files in the encoders/ directory 26 | for file in sorted(os.listdir(os.path.dirname(__file__))): 27 | if file.endswith(".py") and not file.startswith("_"): 28 | module = file[: file.find(".py")] 29 | importlib.import_module("fairseq.data.encoders." + module) 30 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/encoders/bytes.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from fairseq.data.encoders import register_bpe 8 | from fairseq.data.encoders.byte_utils import ( 9 | SPACE, 10 | SPACE_ESCAPE, 11 | byte_encode, 12 | smart_byte_decode, 13 | ) 14 | 15 | 16 | @register_bpe("bytes") 17 | class Bytes(object): 18 | def __init__(self, *unused): 19 | pass 20 | 21 | @staticmethod 22 | def add_args(parser): 23 | pass 24 | 25 | @staticmethod 26 | def encode(x: str) -> str: 27 | encoded = byte_encode(x) 28 | escaped = encoded.replace(SPACE, SPACE_ESCAPE) 29 | return SPACE.join(list(escaped)) 30 | 31 | @staticmethod 32 | def decode(x: str) -> str: 33 | unescaped = x.replace(SPACE, "").replace(SPACE_ESCAPE, SPACE) 34 | return smart_byte_decode(unescaped) 35 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/encoders/characters.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from fairseq.data.encoders import register_bpe 8 | 9 | 10 | SPACE = chr(32) 11 | SPACE_ESCAPE = chr(9601) 12 | 13 | 14 | @register_bpe("characters") 15 | class Characters(object): 16 | def __init__(self, *unused): 17 | pass 18 | 19 | @staticmethod 20 | def add_args(parser): 21 | pass 22 | 23 | @staticmethod 24 | def encode(x: str) -> str: 25 | escaped = x.replace(SPACE, SPACE_ESCAPE) 26 | return SPACE.join(list(escaped)) 27 | 28 | @staticmethod 29 | def decode(x: str) -> str: 30 | return x.replace(SPACE, "").replace(SPACE_ESCAPE, SPACE) 31 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/encoders/nltk_tokenizer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from fairseq.data.encoders import register_tokenizer 7 | from fairseq.dataclass import FairseqDataclass 8 | 9 | 10 | @register_tokenizer("nltk", dataclass=FairseqDataclass) 11 | class NLTKTokenizer(object): 12 | def __init__(self, *unused): 13 | try: 14 | from nltk.tokenize import word_tokenize 15 | 16 | self.word_tokenize = word_tokenize 17 | except ImportError: 18 | raise ImportError("Please install nltk with: pip install nltk") 19 | 20 | def encode(self, x: str) -> str: 21 | return " ".join(self.word_tokenize(x)) 22 | 23 | def decode(self, x: str) -> str: 24 | return x 25 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/encoders/space_tokenizer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import re 7 | 8 | from fairseq.data.encoders import register_tokenizer 9 | from fairseq.dataclass import FairseqDataclass 10 | 11 | 12 | @register_tokenizer("space", dataclass=FairseqDataclass) 13 | class SpaceTokenizer(object): 14 | def __init__(self, *unused): 15 | self.space_tok = re.compile(r"\s+") 16 | 17 | def encode(self, x: str) -> str: 18 | return self.space_tok.sub(" ", x) 19 | 20 | def decode(self, x: str) -> str: 21 | return x 22 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/encoders/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | from fairseq.data import encoders 8 | 9 | 10 | def get_whole_word_mask(args, dictionary): 11 | bpe = encoders.build_bpe(args) 12 | if bpe is not None: 13 | 14 | def is_beginning_of_word(i): 15 | if i < dictionary.nspecial: 16 | # special elements are always considered beginnings 17 | return True 18 | tok = dictionary[i] 19 | if tok.startswith("madeupword"): 20 | return True 21 | try: 22 | return bpe.is_beginning_of_word(tok) 23 | except ValueError: 24 | return True 25 | 26 | mask_whole_words = torch.ByteTensor( 27 | list(map(is_beginning_of_word, range(len(dictionary)))) 28 | ) 29 | return mask_whole_words 30 | return None 31 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/id_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | 8 | from . import FairseqDataset 9 | 10 | 11 | class IdDataset(FairseqDataset): 12 | def __getitem__(self, index): 13 | return index 14 | 15 | def __len__(self): 16 | return 0 17 | 18 | def collater(self, samples): 19 | return torch.tensor(samples) 20 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/legacy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .block_pair_dataset import BlockPairDataset 7 | from .masked_lm_dataset import MaskedLMDataset 8 | from .masked_lm_dictionary import BertDictionary, MaskedLMDictionary 9 | 10 | 11 | __all__ = [ 12 | "BertDictionary", 13 | "BlockPairDataset", 14 | "MaskedLMDataset", 15 | "MaskedLMDictionary", 16 | ] 17 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/list_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import BaseWrapperDataset 7 | 8 | 9 | class ListDataset(BaseWrapperDataset): 10 | def __init__(self, dataset, sizes=None): 11 | super().__init__(dataset) 12 | self._sizes = sizes 13 | 14 | def __iter__(self): 15 | for x in self.dataset: 16 | yield x 17 | 18 | def collater(self, samples): 19 | return samples 20 | 21 | @property 22 | def sizes(self): 23 | return self._sizes 24 | 25 | def num_tokens(self, index): 26 | return self.sizes[index] 27 | 28 | def size(self, index): 29 | return self.sizes[index] 30 | 31 | def set_epoch(self, epoch): 32 | pass 33 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/lru_cache_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from functools import lru_cache 7 | 8 | from . import BaseWrapperDataset 9 | 10 | 11 | class LRUCacheDataset(BaseWrapperDataset): 12 | def __init__(self, dataset, token=None): 13 | super().__init__(dataset) 14 | 15 | @lru_cache(maxsize=8) 16 | def __getitem__(self, index): 17 | return self.dataset[index] 18 | 19 | @lru_cache(maxsize=8) 20 | def collater(self, samples): 21 | return self.dataset.collater(samples) 22 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/multilingual/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/num_samples_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import FairseqDataset 7 | 8 | 9 | class NumSamplesDataset(FairseqDataset): 10 | def __getitem__(self, index): 11 | return 1 12 | 13 | def __len__(self): 14 | return 0 15 | 16 | def collater(self, samples): 17 | return sum(samples) 18 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/numel_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import numpy as np 7 | import torch 8 | 9 | from . import BaseWrapperDataset 10 | 11 | 12 | class NumelDataset(BaseWrapperDataset): 13 | def __init__(self, dataset, reduce=False): 14 | super().__init__(dataset) 15 | self.reduce = reduce 16 | 17 | def __getitem__(self, index): 18 | item = self.dataset[index] 19 | if torch.is_tensor(item): 20 | return torch.numel(item) 21 | else: 22 | return np.size(item) 23 | 24 | def __len__(self): 25 | return len(self.dataset) 26 | 27 | def collater(self, samples): 28 | if self.reduce: 29 | return sum(samples) 30 | else: 31 | return torch.tensor(samples) 32 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/offset_tokens_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import BaseWrapperDataset 7 | 8 | 9 | class OffsetTokensDataset(BaseWrapperDataset): 10 | def __init__(self, dataset, offset): 11 | super().__init__(dataset) 12 | self.offset = offset 13 | 14 | def __getitem__(self, idx): 15 | return self.dataset[idx] + self.offset 16 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/pad_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from fairseq.data import data_utils 7 | 8 | from . import BaseWrapperDataset 9 | 10 | 11 | class PadDataset(BaseWrapperDataset): 12 | def __init__(self, dataset, pad_idx, left_pad): 13 | super().__init__(dataset) 14 | self.pad_idx = pad_idx 15 | self.left_pad = left_pad 16 | 17 | def collater(self, samples): 18 | return data_utils.collate_tokens(samples, self.pad_idx, left_pad=self.left_pad) 19 | 20 | 21 | class LeftPadDataset(PadDataset): 22 | def __init__(self, dataset, pad_idx): 23 | super().__init__(dataset, pad_idx, left_pad=True) 24 | 25 | 26 | class RightPadDataset(PadDataset): 27 | def __init__(self, dataset, pad_idx): 28 | super().__init__(dataset, pad_idx, left_pad=False) 29 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/prepend_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import numpy as np 7 | import torch 8 | 9 | from . import BaseWrapperDataset 10 | 11 | 12 | class PrependDataset(BaseWrapperDataset): 13 | def __init__(self, dataset, prepend_getter, ensure_first_token_is=None): 14 | super().__init__(dataset) 15 | self.prepend_getter = prepend_getter 16 | self.ensure_first_token = ensure_first_token_is 17 | 18 | def __getitem__(self, idx): 19 | item = self.dataset[idx] 20 | is_tuple = isinstance(item, tuple) 21 | src = item[0] if is_tuple else item 22 | 23 | assert self.ensure_first_token is None or src[0] == self.ensure_first_token 24 | prepend_idx = self.prepend_getter(self.dataset, idx) 25 | assert isinstance(prepend_idx, int) 26 | src[0] = prepend_idx 27 | item = tuple((src,) + item[1:]) if is_tuple else src 28 | return item 29 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/raw_label_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | 8 | from . import FairseqDataset 9 | 10 | 11 | class RawLabelDataset(FairseqDataset): 12 | def __init__(self, labels): 13 | super().__init__() 14 | self.labels = labels 15 | 16 | def __getitem__(self, index): 17 | return self.labels[index] 18 | 19 | def __len__(self): 20 | return len(self.labels) 21 | 22 | def collater(self, samples): 23 | return torch.tensor(samples) 24 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/roll_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | 8 | from . import BaseWrapperDataset 9 | 10 | 11 | class RollDataset(BaseWrapperDataset): 12 | def __init__(self, dataset, shifts): 13 | super().__init__(dataset) 14 | self.shifts = shifts 15 | 16 | def __getitem__(self, index): 17 | item = self.dataset[index] 18 | return torch.roll(item, self.shifts) 19 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/sort_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import numpy as np 7 | 8 | from . import BaseWrapperDataset 9 | 10 | 11 | class SortDataset(BaseWrapperDataset): 12 | def __init__(self, dataset, sort_order): 13 | super().__init__(dataset) 14 | if not isinstance(sort_order, (list, tuple)): 15 | sort_order = [sort_order] 16 | self.sort_order = sort_order 17 | 18 | assert all(len(so) == len(dataset) for so in sort_order) 19 | 20 | def ordered_indices(self): 21 | return np.lexsort(self.sort_order) 22 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/data/strip_token_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import BaseWrapperDataset 7 | 8 | 9 | class StripTokenDataset(BaseWrapperDataset): 10 | def __init__(self, dataset, id_to_strip): 11 | super().__init__(dataset) 12 | self.id_to_strip = id_to_strip 13 | 14 | def __getitem__(self, index): 15 | item = self.dataset[index] 16 | while len(item) > 0 and item[-1] == self.id_to_strip: 17 | item = item[:-1] 18 | while len(item) > 0 and item[0] == self.id_to_strip: 19 | item = item[1:] 20 | return item 21 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/dataclass/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .configs import FairseqDataclass 7 | from .constants import ChoiceEnum 8 | 9 | 10 | __all__ = [ 11 | "FairseqDataclass", 12 | "ChoiceEnum", 13 | ] 14 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/distributed/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .distributed_timeout_wrapper import DistributedTimeoutWrapper 7 | from .fully_sharded_data_parallel import fsdp_enable_wrap, fsdp_wrap, FullyShardedDataParallel 8 | from .legacy_distributed_data_parallel import LegacyDistributedDataParallel 9 | from .module_proxy_wrapper import ModuleProxyWrapper 10 | from .tpu_distributed_data_parallel import TPUDistributedDataParallel 11 | 12 | 13 | __all__ = [ 14 | "DistributedTimeoutWrapper", 15 | "fsdp_enable_wrap", 16 | "fsdp_wrap", 17 | "FullyShardedDataParallel", 18 | "LegacyDistributedDataParallel", 19 | "ModuleProxyWrapper", 20 | "TPUDistributedDataParallel", 21 | ] 22 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/logging/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/fairseq/logging/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/fairseq/model_parallel/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from . import criterions, models, modules # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/model_parallel/criterions/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import importlib 7 | import os 8 | 9 | 10 | # automatically import any Python files in the criterions/ directory 11 | for file in sorted(os.listdir(os.path.dirname(__file__))): 12 | if file.endswith(".py") and not file.startswith("_"): 13 | module = file[: file.find(".py")] 14 | importlib.import_module("fairseq.model_parallel.criterions." + module) 15 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/model_parallel/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import importlib 7 | import os 8 | 9 | 10 | # automatically import any Python files in the models/ directory 11 | models_dir = os.path.dirname(__file__) 12 | for file in os.listdir(models_dir): 13 | path = os.path.join(models_dir, file) 14 | if ( 15 | not file.startswith("_") 16 | and not file.startswith(".") 17 | and (file.endswith(".py") or os.path.isdir(path)) 18 | ): 19 | model_name = file[: file.find(".py")] if file.endswith(".py") else file 20 | module = importlib.import_module("fairseq.model_parallel.models." + model_name) 21 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/model_parallel/models/pipeline_parallel_transformer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .model import * # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/model_parallel/models/roberta/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .model import * # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/model_parallel/modules/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | """isort:skip_file""" 6 | 7 | from .multihead_attention import ModelParallelMultiheadAttention 8 | from .transformer_layer import ( 9 | ModelParallelTransformerEncoderLayer, 10 | ModelParallelTransformerDecoderLayer, 11 | ) 12 | 13 | __all__ = [ 14 | "ModelParallelMultiheadAttention", 15 | "ModelParallelTransformerEncoderLayer", 16 | "ModelParallelTransformerDecoderLayer", 17 | ] 18 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/models/bart/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .hub_interface import * # noqa 7 | from .model import * # noqa 8 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/models/hubert/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .hubert import * # noqa 7 | from .hubert_asr import * # noqa 8 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/models/huggingface/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import importlib 7 | import os 8 | 9 | 10 | # automatically import any Python files in the models/huggingface/ directory 11 | models_dir = os.path.dirname(__file__) 12 | for file in os.listdir(models_dir): 13 | path = os.path.join(models_dir, file) 14 | if ( 15 | not file.startswith("_") 16 | and not file.startswith(".") 17 | and (file.endswith(".py") or os.path.isdir(path)) 18 | ): 19 | model_name = file[: file.find(".py")] if file.endswith(".py") else file 20 | module = importlib.import_module("fairseq.models.huggingface." + model_name) 21 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/models/nat/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | """isort:skip_file""" 6 | 7 | from .fairseq_nat_model import * 8 | from .nonautoregressive_transformer import * 9 | from .nat_crf_transformer import * 10 | from .iterative_nonautoregressive_transformer import * 11 | from .cmlm_transformer import * 12 | from .levenshtein_transformer import * 13 | from .insertion_transformer import * 14 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/models/roberta/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .hub_interface import * # noqa 7 | from .model import * # noqa 8 | from .enc_dec import * # noqa 9 | from .model_camembert import * # noqa 10 | from .model_gottbert import * # noqa 11 | from .model_xlmr import * # noqa 12 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/models/speech_to_text/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .berard import * # noqa 7 | from .convtransformer import * # noqa 8 | from .s2t_transformer import * # noqa 9 | from .xm_transformer import * # noqa 10 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/models/wav2vec/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .wav2vec import * # noqa 7 | from .wav2vec2 import * # noqa 8 | from .wav2vec2_asr import * # noqa 9 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/dynamicconv_layer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .dynamicconv_layer import DynamicconvLayer # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/dynamicconv_layer/dynamiconv_cpu.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | std::vector 5 | dynamicconv_cpu_forward(float* input, float* filters, int padding_l); 6 | 7 | std::vector dynamicconv_cpu_backward( 8 | float* gradOutput, 9 | int padding_l, 10 | float* input, 11 | float* filters); 12 | 13 | std::vector 14 | dynamicconv_forward(float* input, float* filters, int padding_l) { 15 | return dynamicconv_cpu_forward(input, filters, padding_l); 16 | } 17 | 18 | std::vector dynamicconv_backward( 19 | float* gradOutput, 20 | int padding_l, 21 | float* input, 22 | float* filters) { 23 | return dynamicconv_cpu_backward(gradOutput, padding_l, input, filters); 24 | } 25 | 26 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 27 | m.def("forward", &dynamicconv_forward, "dynamicconv forward (CPU)"); 28 | m.def("backward", &dynamicconv_backward, "dynamicconv backward (CPU)"); 29 | } 30 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/dynamicconv_layer/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from setuptools import setup 8 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 9 | 10 | 11 | setup( 12 | name="dynamicconv_layer", 13 | ext_modules=[ 14 | CUDAExtension( 15 | name="dynamicconv_cuda", 16 | sources=[ 17 | "dynamicconv_cuda.cpp", 18 | "dynamicconv_cuda_kernel.cu", 19 | ], 20 | ), 21 | ], 22 | cmdclass={"build_ext": BuildExtension}, 23 | ) 24 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/fp32_group_norm.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | """ 6 | Layer norm done in fp32 (for fp16 training) 7 | """ 8 | 9 | import torch.nn as nn 10 | import torch.nn.functional as F 11 | 12 | 13 | class Fp32GroupNorm(nn.GroupNorm): 14 | def __init__(self, *args, **kwargs): 15 | super().__init__(*args, **kwargs) 16 | 17 | def forward(self, input): 18 | output = F.group_norm( 19 | input.float(), 20 | self.num_groups, 21 | self.weight.float() if self.weight is not None else None, 22 | self.bias.float() if self.bias is not None else None, 23 | self.eps, 24 | ) 25 | return output.type_as(input) 26 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/gelu.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | """ 6 | See "Gaussian Error Linear Units (GELUs)" by Dan Hendrycks and Kevin Gimpel with 7 | the corresponding GitHub repo: https://github.com/hendrycks/GELUs 8 | """ 9 | 10 | import math 11 | 12 | import torch 13 | import torch.nn as nn 14 | 15 | 16 | def gelu_accurate(x): 17 | if not hasattr(gelu_accurate, "_a"): 18 | gelu_accurate._a = math.sqrt(2 / math.pi) 19 | return ( 20 | 0.5 * x * (1 + torch.tanh(gelu_accurate._a * (x + 0.044715 * torch.pow(x, 3)))) 21 | ) 22 | 23 | 24 | def gelu(x: torch.Tensor) -> torch.Tensor: 25 | return torch.nn.functional.gelu(x.float()).type_as(x) 26 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/grad_multiply.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | 8 | 9 | class GradMultiply(torch.autograd.Function): 10 | @staticmethod 11 | def forward(ctx, x, scale): 12 | ctx.scale = scale 13 | res = x.new(x) 14 | return res 15 | 16 | @staticmethod 17 | def backward(ctx, grad): 18 | return grad * ctx.scale, None 19 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/lightconv_layer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .lightconv_layer import LightconvLayer # noqa 7 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/lightconv_layer/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from setuptools import setup 8 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 9 | 10 | 11 | setup( 12 | name="lightconv_layer", 13 | ext_modules=[ 14 | CUDAExtension( 15 | "lightconv_cuda", 16 | [ 17 | "lightconv_cuda.cpp", 18 | "lightconv_cuda_kernel.cu", 19 | ], 20 | ), 21 | ], 22 | cmdclass={"build_ext": BuildExtension}, 23 | ) 24 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/quantization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/fairseq/modules/quantization/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/quantization/pq/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .utils import SizeTracker, get_param, attrsetter, quantize_model_ # NOQA 7 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/quantization/pq/modules/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .qconv import PQConv2d # NOQA 7 | from .qemb import PQEmbedding # NOQA 8 | from .qlinear import PQLinear # NOQA 9 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/quantization/scalar/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .utils import quantize_model_ # NOQA 7 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/quantization/scalar/modules/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from .qact import ActivationQuantizer # NOQA 7 | from .qconv import IntConv2d # NOQA 8 | from .qemb import IntEmbedding # NOQA 9 | from .qlinear import IntLinear # NOQA 10 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/same_pad.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from torch import nn 8 | 9 | 10 | class SamePad(nn.Module): 11 | def __init__(self, kernel_size, causal=False): 12 | super().__init__() 13 | if causal: 14 | self.remove = kernel_size - 1 15 | else: 16 | self.remove = 1 if kernel_size % 2 == 0 else 0 17 | 18 | def forward(self, x): 19 | if self.remove > 0: 20 | x = x[:, :, : -self.remove] 21 | return x 22 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/scalar_bias.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | 7 | import torch 8 | 9 | 10 | class ScalarBias(torch.autograd.Function): 11 | """ 12 | Adds a vector of scalars, used in self-attention mechanism to allow 13 | the model to optionally attend to this vector instead of the past 14 | """ 15 | 16 | @staticmethod 17 | def forward(ctx, input, dim, bias_init): 18 | size = list(input.size()) 19 | size[dim] += 1 20 | output = input.new(*size).fill_(bias_init) 21 | output.narrow(dim, 1, size[dim] - 1).copy_(input) 22 | ctx.dim = dim 23 | return output 24 | 25 | @staticmethod 26 | def backward(ctx, grad): 27 | return grad.narrow(ctx.dim, 1, grad.size(ctx.dim) - 1), None, None 28 | 29 | 30 | def scalar_bias(input, dim, bias_init=0): 31 | return ScalarBias.apply(input, dim, bias_init) 32 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/transpose_last.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | """ 6 | transpose last 2 dimensions of the input 7 | """ 8 | 9 | import torch.nn as nn 10 | 11 | 12 | class TransposeLast(nn.Module): 13 | def __init__(self, deconstruct_idx=None): 14 | super().__init__() 15 | self.deconstruct_idx = deconstruct_idx 16 | 17 | def forward(self, x): 18 | if self.deconstruct_idx is not None: 19 | x = x[self.deconstruct_idx] 20 | return x.transpose(-2, -1) 21 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/modules/unfold.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch.nn.functional as F 7 | 8 | 9 | def unfold1d(x, kernel_size, padding_l, pad_value=0): 10 | """unfold T x B x C to T x B x C x K""" 11 | if kernel_size > 1: 12 | T, B, C = x.size() 13 | x = F.pad( 14 | x, (0, 0, 0, 0, padding_l, kernel_size - 1 - padding_l), value=pad_value 15 | ) 16 | x = x.as_strided((T, B, C, kernel_size), (B * C, C, 1, B * C)) 17 | else: 18 | x = x.unsqueeze(3) 19 | return x 20 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/scoring/chrf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from fairseq.scoring import BaseScorer, register_scorer 7 | 8 | 9 | @register_scorer("chrf") 10 | class ChrFScorer(BaseScorer): 11 | def __init__(self, args): 12 | super(ChrFScorer, self).__init__(args) 13 | import sacrebleu 14 | 15 | self.sacrebleu = sacrebleu 16 | 17 | def add_string(self, ref, pred): 18 | self.ref.append(ref) 19 | self.pred.append(pred) 20 | 21 | def score(self, order=4): 22 | return self.result_string(order).score 23 | 24 | def result_string(self, order=4): 25 | if order != 4: 26 | raise NotImplementedError 27 | return self.sacrebleu.corpus_chrf(self.pred, [self.ref]).format() 28 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/tokenizer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import re 7 | 8 | 9 | SPACE_NORMALIZER = re.compile(r"\s+") 10 | 11 | 12 | def tokenize_line(line): 13 | line = SPACE_NORMALIZER.sub(" ", line) 14 | line = line.strip() 15 | return line.split() 16 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq/version.txt: -------------------------------------------------------------------------------- 1 | 1.0.0a0 2 | -------------------------------------------------------------------------------- /Machine_Translation/fairseq_cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/fairseq_cli/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel", "cython"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /Machine_Translation/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/scripts/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/scripts/compound_split_bleu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -ne 1 ]; then 4 | echo "usage: $0 GENERATE_PY_OUTPUT" 5 | exit 1 6 | fi 7 | 8 | GEN=$1 9 | 10 | SYS=$GEN.sys 11 | REF=$GEN.ref 12 | 13 | if [ $(tail -n 1 $GEN | grep BLEU | wc -l) -ne 1 ]; then 14 | echo "not done generating" 15 | exit 16 | fi 17 | 18 | grep ^H $GEN | awk -F '\t' '{print $NF}' | perl -ple 's{(\S)-(\S)}{$1 ##AT##-##AT## $2}g' > $SYS 19 | grep ^T $GEN | cut -f2- | perl -ple 's{(\S)-(\S)}{$1 ##AT##-##AT## $2}g' > $REF 20 | fairseq-score --sys $SYS --ref $REF 21 | -------------------------------------------------------------------------------- /Machine_Translation/scripts/constraints/validate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) Facebook, Inc. and its affiliates. 4 | # 5 | # This source code is licensed under the MIT license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | import sys 9 | 10 | 11 | """Reads in a fairseq output file, and verifies that the constraints 12 | (C- lines) are present in the output (the first H- line). Assumes that 13 | constraints are listed prior to the first hypothesis. 14 | """ 15 | 16 | constraints = [] 17 | found = 0 18 | total = 0 19 | for line in sys.stdin: 20 | if line.startswith("C-"): 21 | constraints.append(line.rstrip().split("\t")[1]) 22 | elif line.startswith("H-"): 23 | text = line.split("\t")[2] 24 | 25 | for constraint in constraints: 26 | total += 1 27 | if constraint in text: 28 | found += 1 29 | else: 30 | print(f"No {constraint} in {text}", file=sys.stderr) 31 | 32 | constraints = [] 33 | 34 | print(f"Found {found} / {total} = {100 * found / total:.1f}%") 35 | -------------------------------------------------------------------------------- /Machine_Translation/scripts/convert_dictionary.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Facebook, Inc. and its affiliates. 2 | -- 3 | -- This source code is licensed under the MIT license found in the 4 | -- LICENSE file in the root directory of this source tree. 5 | -- 6 | -- Usage: convert_dictionary.lua 7 | require 'fairseq' 8 | require 'torch' 9 | require 'paths' 10 | 11 | if #arg < 1 then 12 | print('usage: convert_dictionary.lua ') 13 | os.exit(1) 14 | end 15 | if not paths.filep(arg[1]) then 16 | print('error: file does not exit: ' .. arg[1]) 17 | os.exit(1) 18 | end 19 | 20 | dict = torch.load(arg[1]) 21 | dst = paths.basename(arg[1]):gsub('.th7', '.txt') 22 | assert(dst:match('.txt$')) 23 | 24 | f = io.open(dst, 'w') 25 | for idx, symbol in ipairs(dict.index_to_symbol) do 26 | if idx > dict.cutoff then 27 | break 28 | end 29 | f:write(symbol) 30 | f:write(' ') 31 | f:write(dict.index_to_freq[idx]) 32 | f:write('\n') 33 | end 34 | f:close() 35 | -------------------------------------------------------------------------------- /Machine_Translation/scripts/sacrebleu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -ne 4 ]; then 4 | echo "usage: $0 TESTSET SRCLANG TGTLANG GEN" 5 | exit 1 6 | fi 7 | 8 | TESTSET=$1 9 | SRCLANG=$2 10 | TGTLANG=$3 11 | 12 | GEN=$4 13 | 14 | if ! command -v sacremoses &> /dev/null 15 | then 16 | echo "sacremoses could not be found, please install with: pip install sacremoses" 17 | exit 18 | fi 19 | 20 | grep ^H $GEN \ 21 | | sed 's/^H\-//' \ 22 | | sort -n -k 1 \ 23 | | cut -f 3 \ 24 | | sacremoses detokenize \ 25 | > $GEN.sorted.detok 26 | 27 | sacrebleu --test-set $TESTSET --language-pair "${SRCLANG}-${TGTLANG}" < $GEN.sorted.detok 28 | -------------------------------------------------------------------------------- /Machine_Translation/scripts/spm_train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | from __future__ import absolute_import, division, print_function, unicode_literals 9 | 10 | import sys 11 | 12 | import sentencepiece as spm 13 | 14 | 15 | if __name__ == "__main__": 16 | spm.SentencePieceTrainer.Train(" ".join(sys.argv[1:])) 17 | -------------------------------------------------------------------------------- /Machine_Translation/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/tests/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/tests/distributed/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/tests/distributed/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/tests/gpu/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/tests/gpu/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/tests/gpu/transformer_quantization_config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # This file defines example configuration arguments for quantizing 7 | # a transformer model with product quantization 8 | 9 | n_centroids: 10 | Linear: 11 | key: in_features 12 | value: {"*": 8} 13 | Embedding: 14 | key: embedding_dim 15 | value: {"*": 8} 16 | 17 | block_sizes: 18 | Linear: 19 | key: fuzzy_name 20 | value: {fc: 8, attn: 4, emb: 4} 21 | Embedding: 22 | key: fuzzy_name 23 | value: {emb: 8} 24 | 25 | layers_to_quantize: 26 | - decoder\\.layers\\.\d+\\.fc[12] 27 | - decoder\\.embed_tokens\\.embeddings\\.[012]\\.[01] 28 | - decoder\\.layers\\.\d+\\.self_attn\\.(k_proj|v_proj|q_proj|out_proj) 29 | -------------------------------------------------------------------------------- /Machine_Translation/tests/speech_recognition/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/tests/speech_recognition/__init__.py -------------------------------------------------------------------------------- /Machine_Translation/tests/test_iopath.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import unittest 7 | from unittest import mock 8 | 9 | 10 | class TestIOPath(unittest.TestCase): 11 | 12 | def test_no_iopath(self): 13 | from .test_reproducibility import TestReproducibility 14 | 15 | with mock.patch.dict("sys.modules", {"iopath": None}): 16 | # reuse reproducibility tests, which are e2e tests that should cover 17 | # most checkpoint related functionality 18 | TestReproducibility._test_reproducibility(self, "test_reproducibility") 19 | 20 | def test_no_supports_rename(self): 21 | from .test_reproducibility import TestReproducibility 22 | 23 | with mock.patch("fairseq.file_io.PathManager.supports_rename") as mock_fn: 24 | mock_fn.return_value = False 25 | TestReproducibility._test_reproducibility(self, "test_reproducibility") 26 | 27 | 28 | if __name__ == "__main__": 29 | unittest.main() 30 | -------------------------------------------------------------------------------- /Machine_Translation/train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 -u 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | """ 7 | Legacy entry point. Use fairseq_cli/train.py or fairseq-train instead. 8 | """ 9 | 10 | from fairseq_cli.train import cli_main 11 | 12 | 13 | if __name__ == "__main__": 14 | cli_main() 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Contrastive-Disentangled-Memory-for-sequential-Learning 2 | 3 | # Introduction 4 | 5 | This is a source code for Transformer with a contrastive disentangled memory. 6 | The implementation includes machine translation and online speech recognition. 7 | 8 | 9 | ## Requirements and Frameworks 10 | - Python 3.6.1+ 11 | - Fairseq 12 | - ESPnet 13 | 14 | ### To do machine translation 15 | ```sh 16 | cd Machine_Translation 17 | ``` 18 | 19 | ### To do speech recognition 20 | ```sh 21 | cd Speech_Recognition 22 | ``` 23 | -------------------------------------------------------------------------------- /Speech_Recognition/README.md: -------------------------------------------------------------------------------- 1 | ## Python Requirement 2 | 3 | ```sh 4 | torch 5 | chainer==6.0.0 cupy==6.0.0 6 | torchaudio 7 | torch_optimizer 8 | ``` 9 | 10 | ## Setup 11 | 12 | ```sh 13 | cd tools 14 | make 15 | ``` 16 | 17 | ## Download Aishell dadaset and Preprocessing 18 | 19 | ```sh 20 | cd egs/aishell/asr 21 | ./run.sh --stage -1 22 | ``` 23 | 24 | ## Train 25 | 26 | ```sh 27 | cd egs/aishell/asr 28 | ./run.sh --stage 4 29 | ``` 30 | 31 | ## Test 32 | 33 | ```sh 34 | cd egs/aishell/asr 35 | ./run.sh --stage 5 36 | ``` 37 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/RESULTS.md: -------------------------------------------------------------------------------- 1 | # Result 2 | - [Online compressive transformer version 1](https://drive.google.com/drive/folders/1wmG7B2Ld0k8B9aYqYJU7lhS9DiwNqNKq?usp=sharing) 3 | - [Online compressive transformer version 2](https://drive.google.com/drive/folders/1Ex2qypmdnzLJDzH8LyXzbbfyuq4yVrhN?usp=sharing) 4 | - [Synchronous transformer[1]](https://drive.google.com/drive/folders/1_OyG2p5EikSRP_T37odkH0-EHIdpDG-F?usp=sharing) 5 | - [Transformer](https://drive.google.com/drive/folders/1pVspTH4ljY4ddYujlSo4PRKjDICC1reV?usp=sharing) 6 | 7 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/decode.yaml: -------------------------------------------------------------------------------- 1 | batchsize: 0 2 | beam-size: 5 3 | penalty: 0.0 4 | maxlenratio: 0.0 5 | minlenratio: 0.0 6 | ctc-weight: 0.0 7 | #lm-weight: 0.7 8 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/decode_memory_transformer.yaml: -------------------------------------------------------------------------------- 1 | # decoding parameters 2 | batch: 0 3 | score-norm-transducer: true 4 | beam-size: 5 5 | ctc-weight: 0 6 | # lm-weight: 0.05 -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/decode_sync_transformer.yaml: -------------------------------------------------------------------------------- 1 | # decoding parameters 2 | batch: 0 3 | score-norm-transducer: true 4 | beam-size: 5 5 | # lm-weight: 0.05 -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/fbank.conf: -------------------------------------------------------------------------------- 1 | --sample-frequency=16000 2 | --num-mel-bins=80 3 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/gpu.conf: -------------------------------------------------------------------------------- 1 | # Default configuration 2 | command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64* 3 | option mem=* -l mem_free=$0,ram_free=$0 4 | option mem=0 # Do not add anything to qsub_opts 5 | option num_threads=* -pe smp $0 6 | option num_threads=1 # Do not add anything to qsub_opts 7 | option max_jobs_run=* -tc $0 8 | default gpu=0 9 | option gpu=0 10 | option gpu=* -l 'hostname=b1[12345678]*|c*,gpu=$0' -q g.q -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/lm.yaml: -------------------------------------------------------------------------------- 1 | # rnnlm related 2 | layer: 2 3 | unit: 650 4 | opt: sgd # or adam 5 | batchsize: 64 # batch size in LM training 6 | epoch: 20 # if the data size is large, we can reduce this 7 | patience: 3 8 | maxlen: 100 # if sentence length > lm_maxlen, lm_batchsize is automatically reduced 9 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/pitch.conf: -------------------------------------------------------------------------------- 1 | --sample-frequency=16000 2 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/queue.conf: -------------------------------------------------------------------------------- 1 | # Default configuration 2 | command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64* 3 | option mem=* -l mem_free=$0,ram_free=$0 4 | option mem=0 # Do not add anything to qsub_opts 5 | option num_threads=* -pe smp $0 6 | option num_threads=1 # Do not add anything to qsub_opts 7 | option max_jobs_run=* -tc $0 8 | default gpu=0 9 | option gpu=0 10 | option gpu=* -l gpu=$0 -q g.q 11 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/slurm.conf: -------------------------------------------------------------------------------- 1 | # Default configuration 2 | command sbatch --export=PATH --ntasks-per-node=1 3 | option time=* --time $0 4 | option mem=* --mem-per-cpu $0 5 | option mem=0 # Do not add anything to qsub_opts 6 | option num_threads=* --cpus-per-task $0 --ntasks-per-node=1 7 | option num_threads=1 --cpus-per-task 1 --ntasks-per-node=1 # Do not add anything to qsub_opts 8 | default gpu=0 9 | option gpu=0 -p cpu 10 | option gpu=* -p gpu --gres=gpu:$0 11 | # note: the --max-jobs-run option is supported as a special case 12 | # by slurm.pl and you don't have to handle it in the config file. 13 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/specaug.yaml: -------------------------------------------------------------------------------- 1 | process: 2 | # these three processes are a.k.a. SpecAugument 3 | - type: "time_warp" 4 | max_time_warp: 5 5 | inplace: true 6 | mode: "PIL" 7 | - type: "freq_mask" 8 | F: 30 9 | n_mask: 2 10 | inplace: true 11 | replace_with_zero: false 12 | - type: "time_mask" 13 | T: 40 14 | n_mask: 2 15 | inplace: true 16 | replace_with_zero: false 17 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/train.yaml: -------------------------------------------------------------------------------- 1 | # network architecture 2 | # encoder related 3 | elayers: 12 4 | eunits: 2048 5 | # decoder related 6 | dlayers: 6 7 | dunits: 2048 8 | # attention related 9 | adim: 256 10 | aheads: 4 11 | 12 | # hybrid CTC/attention 13 | mtlalpha: 0.3 14 | 15 | # label smoothing 16 | lsm-weight: 0.1 17 | 18 | # minibatch related 19 | batch-size: 32 20 | maxlen-in: 512 # if input length > maxlen-in, batchsize is automatically reduced 21 | maxlen-out: 150 # if output length > maxlen-out, batchsize is automatically reduced 22 | 23 | # optimization related 24 | sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs 25 | opt: noam 26 | accum-grad: 2 27 | grad-clip: 5 28 | patience: 0 29 | epochs: 50 30 | dropout-rate: 0.1 31 | 32 | # transformer specific setting 33 | backend: pytorch 34 | model-module: "espnet.nets.pytorch_backend.e2e_asr_transformer:E2E" 35 | transformer-input-layer: conv2d # encoder architecture type 36 | transformer-lr: 1.0 37 | transformer-warmup-steps: 25000 38 | transformer-attn-dropout-rate: 0.0 39 | transformer-length-normalized-loss: false 40 | transformer-init: pytorch 41 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/train_transducer.yaml: -------------------------------------------------------------------------------- 1 | # minibatch related 2 | batch-size: 20 3 | maxlen-in: 800 4 | maxlen-out: 150 5 | 6 | # optimization related 7 | criterion: loss 8 | early-stop-criterion: "validation/main/loss" 9 | sortagrad: 0 10 | opt: noam 11 | epochs: 15 12 | patience: 0 13 | accum-grad: 4 14 | grad-clip: 5.0 15 | 16 | # transformer related 17 | transformer-lr: 10.0 18 | transformer-warmup-steps: 8000 19 | transformer-attn-dropout-rate-encoder: 0.4 20 | transformer-attn-dropout-rate-decoder: 0.1 21 | 22 | # network architecture 23 | ## encoder related 24 | etype: transformer 25 | transformer-input-layer: conv2d 26 | elayers: 6 27 | eunits: 320 28 | dropout-rate: 0.4 29 | ## decoder related 30 | dtype: transformer 31 | dlayers: 6 32 | dec-embed-dim: 300 33 | dunits: 300 34 | dropout-rate-decoder: 0.1 35 | ## attention related 36 | adim: 320 37 | aheads: 4 38 | ## joint network related 39 | joint-dim: 300 40 | 41 | # transducer related 42 | mtlalpha: 1.0 # mtlalpha should be set to 1.0 (CTC) to use transducer 43 | rnnt-mode: 'rnnt' # switch to 'rnnt-att' to use transducer with attention 44 | model-module: espnet.nets.pytorch_backend.e2e_asr_transducer:E2E -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/tuning/decode_pytorch_transformer.yaml: -------------------------------------------------------------------------------- 1 | batchsize: 0 2 | beam-size: 5 3 | penalty: 0.0 4 | maxlenratio: 0.0 5 | minlenratio: 0.0 6 | ctc-weight: 0.0 7 | #lm-weight: 0.7 8 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/tuning/decode_rnn.yaml: -------------------------------------------------------------------------------- 1 | beam-size: 20 2 | penalty: 0.0 3 | maxlenratio: 0.0 4 | minlenratio: 0.0 5 | ctc-weight: 0.6 6 | lm-weight: 0.3 7 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/conf/tuning/train_rnn.yaml: -------------------------------------------------------------------------------- 1 | # network architecture 2 | # encoder related 3 | etype: vggblstm # encoder architecture type 4 | elayers: 3 5 | eunits: 1024 6 | eprojs: 1024 7 | subsample: "1_2_2_1_1" # skip every n frame from input to nth layers 8 | # decoder related 9 | dlayers: 2 10 | dunits: 1024 11 | # attention related 12 | atype: location 13 | adim: 1024 14 | aconv-chans: 10 15 | aconv-filts: 100 16 | 17 | # hybrid CTC/attention 18 | mtlalpha: 0.5 19 | 20 | # minibatch related 21 | batch-size: 30 22 | maxlen-in: 800 # if input length > maxlen_in, batchsize is automatically reduced 23 | maxlen-out: 150 # if output length > maxlen_out, batchsize is automatically reduced 24 | 25 | # optimization related 26 | opt: adadelta 27 | epochs: 10 28 | patience: 0 29 | 30 | # scheduled sampling option 31 | sampling-probability: 0.0 32 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/local/prepare_for_eer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2015 David Snyder 4 | # Apache 2.0. 5 | # 6 | # Copied from egs/sre10/v1/local/prepare_for_eer.py (commit 9cb4c4c2fb0223ee90c38d98af11305074eb7ef8) 7 | # 8 | # Given a trials and scores file, this script 9 | # prepares input for the binary compute-eer. 10 | import sys 11 | trials = open(sys.argv[1], 'r').readlines() 12 | scores = open(sys.argv[2], 'r').readlines() 13 | spkrutt2target = {} 14 | for line in trials: 15 | spkr, utt, target = line.strip().split() 16 | spkrutt2target[spkr+utt]=target 17 | for line in scores: 18 | spkr, utt, score = line.strip().split() 19 | print("{} {}".format(score, spkrutt2target[spkr+utt])) 20 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/path.sh: -------------------------------------------------------------------------------- 1 | MAIN_ROOT=$PWD/../../.. 2 | KALDI_ROOT=$MAIN_ROOT/tools/kaldi 3 | 4 | [ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh 5 | export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/tools/sctk/bin:$PWD:$PATH 6 | [ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1 7 | . $KALDI_ROOT/tools/config/common_path.sh 8 | export LC_ALL=C 9 | 10 | export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$MAIN_ROOT/tools/chainer_ctc/ext/warp-ctc/build 11 | 12 | #. /home/container_user/hang/espnet/tools/activate_python.sh 13 | if [ -e $MAIN_ROOT/tools/venv/etc/profile.d/conda.sh ]; then #venv 14 | source $MAIN_ROOT/tools/venv/etc/profile.d/conda.sh && conda deactivate && conda activate base #abc 15 | else 16 | source $MAIN_ROOT/tools/venv/bin/activate 17 | fi 18 | 19 | export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH 20 | 21 | export OMP_NUM_THREADS=1 22 | 23 | # NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C 24 | export PYTHONIOENCODING=UTF-8 25 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/chain/e2e/README.txt: -------------------------------------------------------------------------------- 1 | The scripts related to end2end chain training are in this directory 2 | Currently it has 3 scripts: 3 | 4 | ** prepare_e2e.sh which is almost equivalent 5 | to regular chain's build-tree.sh (i.e. it creates the tree and 6 | the transition-model) except it does not require any previously 7 | trained models (in other terms, it does what stages -3 and -2 8 | of steps/train_mono.sh do). 9 | 10 | ** get_egs_e2e.sh: this is simlilar to chain/get_egs.sh except it 11 | uses training FSTs (instead of lattices) to generate end2end egs. 12 | 13 | ** train_e2e.py: this is very similar to chain/train.py but 14 | with fewer stages (e.g. it does not compute the preconditioning matrix) 15 | 16 | 17 | For details please see the comments at top of local/chain/e2e/run_flatstart_*.sh 18 | and also src/chain/chain-generic-numerator.h. 19 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/data/data_dir_manipulation_lib.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | def RunKaldiCommand(command, wait = True): 4 | """ Runs commands frequently seen in Kaldi scripts. These are usually a 5 | sequence of commands connected by pipes, so we use shell=True """ 6 | #logger.info("Running the command\n{0}".format(command)) 7 | p = subprocess.Popen(command, shell = True, 8 | stdout = subprocess.PIPE, 9 | stderr = subprocess.PIPE) 10 | 11 | if wait: 12 | [stdout, stderr] = p.communicate() 13 | if p.returncode is not 0: 14 | raise Exception("There was an error while running the command {0}\n------------\n{1}".format(command, stderr)) 15 | return stdout, stderr 16 | else: 17 | return p 18 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/libs/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Copyright 2016 Vimal Manohar 4 | # Apache 2.0. 5 | 6 | """ This package contains modules and subpackages used in kaldi scripts. 7 | """ 8 | 9 | from . import common 10 | 11 | __all__ = ["common"] 12 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Copyright 2016 Johns Hopkins University (Dan Povey) 4 | # 2016 Vimal Manohar 5 | # 2016 Vijayaditya Peddinti 6 | # 2016 Yiming Wang 7 | # Apache 2.0. 8 | 9 | 10 | # This module has the python functions which facilitate the use of nnet3 toolkit 11 | # It has two sub-modules 12 | # xconfig : Library for parsing high level description of neural networks 13 | # train : Library for training scripts 14 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/report/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Copyright 2016 Vimal Manohar 4 | # Apache 2.0. 5 | 6 | from . import log_parse 7 | 8 | __all__ = ["log_parse"] 9 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/train/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2016 Vimal Manohar 3 | # Apache 2.0 4 | 5 | """ This library has classes and methods commonly used for training nnet3 6 | neural networks. 7 | 8 | It has separate submodules for frame-level objectives and chain objective: 9 | frame_level_objf -- For both recurrent and non-recurrent architectures 10 | chain_objf -- LF-MMI objective training 11 | """ 12 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/train/chain_objf/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Copyright 2016 Vimal Manohar 4 | # Apache 2.0. 5 | 6 | """ This is a subpackage containing modules for training of 7 | deep neural network acoustic model with chain objective. 8 | """ 9 | 10 | from . import acoustic_model 11 | 12 | __all__ = ["acoustic_model"] 13 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/train/frame_level_objf/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Copyright 2016 Vimal Manohar 4 | # Apache 2.0 5 | 6 | """ This library has classes and methods commonly used for training nnet3 7 | neural networks with frame-level objectives. 8 | """ 9 | 10 | from . import common 11 | from . import raw_model 12 | from . import acoustic_model 13 | 14 | __all__ = ["common", "raw_model", "acoustic_model"] 15 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/xconfig/layers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Johns Hopkins University (Dan Povey) 2 | # 2016 Vijayaditya Peddinti 3 | # 2016 Yiming Wang 4 | # Apache 2.0. 5 | 6 | from .basic_layers import * 7 | from .convolution import * 8 | from .attention import * 9 | from .lstm import * 10 | from .gru import * 11 | from .stats_layer import * 12 | from .trivial_layers import * 13 | from .composite_layers import * 14 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/nnet2/get_num_frames.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script works out the approximate number of frames in a training directory. 4 | # This is sometimes needed by higher-level scripts 5 | 6 | 7 | if [ -f path.sh ]; then . ./path.sh; fi 8 | . parse_options.sh || exit 1; 9 | 10 | if [ $# -ne 1 ]; then 11 | ( 12 | echo "Usage: $0 " 13 | echo "Prints the number of frames of data in the data-dir" 14 | ) 1>&2 15 | fi 16 | 17 | data=$1 18 | 19 | if [ ! -f $data/utt2dur ]; then 20 | utils/data/get_utt2dur.sh $data 1>&2 || exit 1 21 | fi 22 | 23 | frame_shift=$(utils/data/get_frame_shift.sh $data) || exit 1 24 | 25 | awk -v s=$frame_shift '{n += $2} END{printf("%.0f\n", (n / s))}' <$data/utt2dur 26 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/nnet3/chain/e2e/README.txt: -------------------------------------------------------------------------------- 1 | The scripts related to end2end chain training are in this directory 2 | Currently it has 3 scripts: 3 | 4 | ** prepare_e2e.sh which is almost equivalent 5 | to regular chain's build-tree.sh (i.e. it creates the tree and 6 | the transition-model) except it does not require any previously 7 | trained models (in other terms, it does what stages -3 and -2 8 | of steps/train_mono.sh do). 9 | 10 | ** get_egs_e2e.sh: this is simlilar to chain/get_egs.sh except it 11 | uses training FSTs (instead of lattices) to generate end2end egs. 12 | 13 | ** train_e2e.py: this is very similar to chain/train.py but 14 | with fewer stages (e.g. it does not compute the preconditioning matrix) 15 | 16 | 17 | For details please see the comments at top of local/chain/e2e/run_flatstart_*.sh 18 | and also src/chain/chain-generic-numerator.h. 19 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/tfrnnlm/check_py.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/steps/tfrnnlm/check_tensorflow_installed.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this script checks if TF is installed to be used with python 4 | # and if TF related binaries in kaldi is ready to use 5 | . ./path.sh 6 | 7 | if which lattice-lmrescore-tf-rnnlm 2>&1>/dev/null; then 8 | echo TensorFlow relate binaries found. This is good. 9 | else 10 | echo TF related binaries not compiled. 11 | echo You need to go to tools/ and run extras/install_tensorflow_cc.sh first 12 | echo and then do \"make\" under both src/tfrnnlm and src/tfrnnlmbin 13 | exit 1 14 | fi 15 | 16 | echo 17 | 18 | if python steps/tfrnnlm/check_py.py 2>/dev/null; then 19 | echo TensorFlow ready to use on the python side. This is good. 20 | else 21 | echo TensorFlow not found on the python side. 22 | echo Please go to tools/ and run extras/install_tensorflow_py.sh to install it 23 | echo If you already have TensorFlow installed somewhere else, you would need 24 | echo to add it to your PATH 25 | exit 1 26 | fi 27 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/ctm/fix_ctm.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | stmfile=$1 4 | ctmfile=$2 5 | 6 | segments_stm=`cat $stmfile | cut -f 1 -d ' ' | sort -u` 7 | segments_ctm=`cat $ctmfile | cut -f 1 -d ' ' | sort -u` 8 | 9 | segments_stm_count=`echo "$segments_stm" | wc -l ` 10 | segments_ctm_count=`echo "$segments_ctm" | wc -l ` 11 | 12 | #echo $segments_stm_count 13 | #echo $segments_ctm_count 14 | 15 | if [ "$segments_stm_count" -gt "$segments_ctm_count" ] ; then 16 | pp=$( diff <(echo "$segments_stm") <(echo "$segments_ctm" ) | grep "^<" | sed "s/^< *//g") 17 | ( 18 | for elem in $pp ; do 19 | echo "$elem 1 0 0 EMPTY_RECOGNIZED_PHRASE" 20 | done 21 | ) >> $ctmfile 22 | echo "FIXED CTM FILE" 23 | exit 0 24 | elif [ "$segments_stm_count" -lt "$segments_ctm_count" ] ; then 25 | echo "Segment STM count: $segments_stm_count" 26 | echo "Segment CTM count: $segments_ctm_count" 27 | echo "FAILURE FIXING CTM FILE" 28 | exit 1 29 | else 30 | exit 0 31 | fi 32 | 33 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/data/get_num_frames.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script works out the approximate number of frames in a training directory. 4 | # This is sometimes needed by higher-level scripts 5 | 6 | 7 | if [ -f path.sh ]; then . ./path.sh; fi 8 | . parse_options.sh || exit 1; 9 | 10 | if [ $# -ne 1 ]; then 11 | ( 12 | echo "Usage: $0 " 13 | echo "Prints the number of frames of data in the data-dir" 14 | ) 1>&2 15 | fi 16 | 17 | data=$1 18 | 19 | if [ ! -f $data/utt2dur ]; then 20 | utils/data/get_utt2dur.sh $data 1>&2 || exit 1 21 | fi 22 | 23 | frame_shift=$(utils/data/get_frame_shift.sh $data) || exit 1 24 | 25 | awk -v s=$frame_shift '{n += $2} END{printf("%.0f\n", (n / s))}' <$data/utt2dur 26 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/data/get_reco2utt_for_data.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Copyright 2016 Vimal Manohar 4 | # Apache 2.0 5 | 6 | if [ $# -ne 1 ]; then 7 | echo "This script outputs a mapping from recording to a list of utterances " 8 | echo "corresponding to the recording. It is analogous to the content of " 9 | echo "a spk2utt file, but is indexed by recording instead of speaker." 10 | echo "Usage: get_reco2utt.sh " 11 | echo " e.g.: get_reco2utt.sh data/train" 12 | exit 1 13 | fi 14 | 15 | data=$1 16 | 17 | if [ ! -s $data/segments ]; then 18 | utils/data/get_segments_for_data.sh $data > $data/segments 19 | fi 20 | 21 | cut -d ' ' -f 1,2 $data/segments | utils/utt2spk_to_spk2utt.pl 22 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/data/get_segments_for_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script operates on a data directory, such as in data/train/, 4 | # and writes new segments to stdout. The file 'segments' maps from 5 | # utterance to time offsets into a recording, with the format: 6 | # 7 | # This script assumes utterance and recording ids are the same (i.e., that 8 | # wav.scp is indexed by utterance), and uses durations from 'utt2dur', 9 | # created if necessary by get_utt2dur.sh. 10 | 11 | . ./path.sh 12 | 13 | if [ $# != 1 ]; then 14 | echo "Usage: $0 [options] " 15 | echo "e.g.:" 16 | echo " $0 data/train > data/train/segments" 17 | exit 1 18 | fi 19 | 20 | data=$1 21 | 22 | if [ ! -s $data/utt2dur ]; then 23 | utils/data/get_utt2dur.sh $data 1>&2 || exit 1; 24 | fi 25 | 26 | # 0 27 | awk '{ print $1, $1, 0, $2 }' $data/utt2dur 28 | 29 | exit 0 30 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/eps2disambig.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | # Copyright 2010-2011 Microsoft Corporation 3 | # 2015 Guoguo Chen 4 | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | # KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED 13 | # WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, 14 | # MERCHANTABLITY OR NON-INFRINGEMENT. 15 | # See the Apache 2 License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This script replaces epsilon with #0 on the input side only, of the G.fst 19 | # acceptor. 20 | 21 | while(<>){ 22 | if (/\s+#0\s+/) { 23 | print STDERR "$0: ERROR: LM has word #0, " . 24 | "which is reserved as disambiguation symbol\n"; 25 | exit 1; 26 | } 27 | s:^(\d+\s+\d+\s+)\(\s+):$1#0$2:; 28 | print; 29 | } 30 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/filt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Apache 2.0 4 | 5 | from __future__ import print_function 6 | import sys 7 | 8 | vocab=set() 9 | with open(sys.argv[1]) as vocabfile: 10 | for line in vocabfile: 11 | vocab.add(line.strip()) 12 | 13 | with open(sys.argv[2]) as textfile: 14 | for line in textfile: 15 | print(" ".join([word if word in vocab else '' for word in line.strip().split()])) 16 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/fix_ctm.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | stmfile=$1 4 | ctmfile=$2 5 | 6 | segments_stm=`cat $stmfile | cut -f 1 -d ' ' | sort -u` 7 | segments_ctm=`cat $ctmfile | cut -f 1 -d ' ' | sort -u` 8 | 9 | segments_stm_count=`echo "$segments_stm" | wc -l ` 10 | segments_ctm_count=`echo "$segments_ctm" | wc -l ` 11 | 12 | #echo $segments_stm_count 13 | #echo $segments_ctm_count 14 | 15 | if [ "$segments_stm_count" -gt "$segments_ctm_count" ] ; then 16 | pp=$( diff <(echo "$segments_stm") <(echo "$segments_ctm" ) | grep "^<" | sed "s/^< *//g") 17 | ( 18 | for elem in $pp ; do 19 | echo "$elem 1 0 0 EMPTY_RECOGNIZED_PHRASE" 20 | done 21 | ) >> $ctmfile 22 | echo "FIXED CTM FILE" 23 | exit 0 24 | elif [ "$segments_stm_count" -lt "$segments_ctm_count" ] ; then 25 | echo "Segment STM count: $segments_stm_count" 26 | echo "Segment CTM count: $segments_ctm_count" 27 | echo "FAILURE FIXING CTM FILE" 28 | exit 1 29 | else 30 | exit 0 31 | fi 32 | 33 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/lang/bpe/prepend_words.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # This script, prepend '|' to every words in the transcript to mark 4 | # the beginning of the words for finding the initial-space of every word 5 | # after decoding. 6 | 7 | import sys 8 | import io 9 | import re 10 | 11 | whitespace = re.compile("[ \t]+") 12 | infile = io.TextIOWrapper(sys.stdin.buffer, encoding='latin-1') 13 | output = io.TextIOWrapper(sys.stdout.buffer, encoding='latin-1') 14 | for line in infile: 15 | words = whitespace.split(line.strip(" \t\r\n")) 16 | output.write(' '.join([ "|"+word for word in words]) + '\n') 17 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/lang/bpe/reverse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # This script, reverse all latin and digits sequences 5 | # (including words like MP3) to put them in the right order in the images. 6 | 7 | import re, os, sys, io 8 | 9 | in_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8') 10 | out_stream = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') 11 | for line in in_stream: 12 | out_stream.write(re.sub(r'[a-zA-Z0-9][a-zA-Z0-9\s\.\:]*[a-zA-Z0-9]', 13 | lambda m:m.group(0)[::-1], line)) 14 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/make_absolute.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script replaces the command readlink -f (which is not portable). 4 | # It turns a pathname into an absolute pathname, including following soft links. 5 | target_file=$1 6 | 7 | cd $(dirname $target_file) 8 | target_file=$(basename $target_file) 9 | 10 | # Iterate down a (possible) chain of symlinks 11 | while [ -L "$target_file" ]; do 12 | target_file=$(readlink $target_file) 13 | cd $(dirname $target_file) 14 | target_file=$(basename $target_file) 15 | done 16 | 17 | # Compute the canonicalized name by finding the physical path 18 | # for the directory we're in and appending the target file. 19 | phys_dir=$(pwd -P) 20 | result=$phys_dir/$target_file 21 | echo $result 22 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/s2eps.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | # Copyright 2010-2011 Microsoft Corporation 3 | 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 11 | # KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED 12 | # WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, 13 | # MERCHANTABLITY OR NON-INFRINGEMENT. 14 | # See the Apache 2 License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script replaces and with (on both input and output sides), 18 | # for the G.fst acceptor. 19 | 20 | while(<>){ 21 | @A = split(" ", $_); 22 | if ( @A >= 4 ) { 23 | if ($A[2] eq "" || $A[2] eq "") { $A[2] = ""; } 24 | if ($A[3] eq "" || $A[3] eq "") { $A[3] = ""; } 25 | } 26 | print join("\t", @A) . "\n"; 27 | } 28 | -------------------------------------------------------------------------------- /Speech_Recognition/egs/aishell/asr/utils/spk2utt_to_utt2spk.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | # Copyright 2010-2011 Microsoft Corporation 3 | 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 11 | # KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED 12 | # WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, 13 | # MERCHANTABLITY OR NON-INFRINGEMENT. 14 | # See the Apache 2 License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | while(<>){ 19 | @A = split(" ", $_); 20 | @A > 1 || die "Invalid line in spk2utt file: $_"; 21 | $s = shift @A; 22 | foreach $u ( @A ) { 23 | print "$u $s\n"; 24 | } 25 | } 26 | 27 | 28 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize main package.""" 2 | 3 | import pkg_resources 4 | 5 | try: 6 | __version__ = pkg_resources.get_distribution("espnet").version 7 | except Exception: 8 | __version__ = "(Not installed from setup.py)" 9 | del pkg_resources 10 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/asr/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/asr/chainer_backend/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/asr/pytorch_backend/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/bin/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/lm/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/lm/chainer_backend/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/lm/pytorch_backend/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/mt/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/mt/pytorch_backend/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/chainer_backend/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/chainer_backend/asr_interface.py: -------------------------------------------------------------------------------- 1 | """ASR Interface module.""" 2 | import chainer 3 | 4 | from espnet.nets.asr_interface import ASRInterface 5 | 6 | 7 | class ChainerASRInterface(ASRInterface, chainer.Chain): 8 | """ASR Interface for ESPnet model implementation.""" 9 | 10 | @staticmethod 11 | def custom_converter(*args, **kw): 12 | """Get customconverter of the model (Chainer only).""" 13 | raise NotImplementedError("custom converter method is not implemented") 14 | 15 | @staticmethod 16 | def custom_updater(*args, **kw): 17 | """Get custom_updater of the model (Chainer only).""" 18 | raise NotImplementedError("custom updater method is not implemented") 19 | 20 | @staticmethod 21 | def custom_parallel_updater(*args, **kw): 22 | """Get custom_parallel_updater of the model (Chainer only).""" 23 | raise NotImplementedError("custom parallel updater method is not implemented") 24 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/chainer_backend/nets_utils.py: -------------------------------------------------------------------------------- 1 | import chainer.functions as F 2 | 3 | 4 | def _subsamplex(x, n): 5 | x = [F.get_item(xx, (slice(None, None, n), slice(None))) for xx in x] 6 | ilens = [xx.shape[0] for xx in x] 7 | return x, ilens 8 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/chainer_backend/rnn/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/chainer_backend/transformer/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/chainer_backend/transformer/layer_norm.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """Class Declaration of Transformer's Label Smootion loss.""" 3 | 4 | import chainer.links as L 5 | 6 | 7 | class LayerNorm(L.LayerNormalization): 8 | """Redirect to L.LayerNormalization.""" 9 | 10 | def __init__(self, dims, eps=1e-12): 11 | """Initialize LayerNorm.""" 12 | super(LayerNorm, self).__init__(size=dims, eps=eps) 13 | 14 | def __call__(self, e): 15 | """Forward LayerNorm.""" 16 | return super(LayerNorm, self).__call__(e) 17 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/chainer_backend/transformer/mask.py: -------------------------------------------------------------------------------- 1 | """Create mask for subsequent steps.""" 2 | 3 | 4 | def make_history_mask(xp, block): 5 | """Prepare the history mask. 6 | 7 | Args: 8 | block (ndarray): Block with dimensions: (B x S). 9 | Returns: 10 | ndarray, np.ndarray: History mask with dimensions (B, S, S). 11 | 12 | """ 13 | batch, length = block.shape 14 | arange = xp.arange(length) 15 | history_mask = (arange[None] <= arange[:, None])[ 16 | None, 17 | ] 18 | history_mask = xp.broadcast_to(history_mask, (batch, length, length)) 19 | return history_mask 20 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/compressive_transducer/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/fastspeech/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/frontends/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/lm/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/rnn/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/streaming/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/sync_transducer/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/tacotron2/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/transducer/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/transformer/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/transformer/add_sos_eos.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright 2019 Shigeki Karita 5 | # Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) 6 | 7 | """Unility funcitons for Transformer.""" 8 | 9 | import torch 10 | 11 | 12 | def add_sos_eos(ys_pad, sos, eos, ignore_id): 13 | """Add and labels. 14 | 15 | :param torch.Tensor ys_pad: batch of padded target sequences (B, Lmax) 16 | :param int sos: index of 17 | :param int eos: index of 18 | :param int ignore_id: index of padding 19 | :return: padded tensor (B, Lmax) 20 | :rtype: torch.Tensor 21 | :return: padded tensor (B, Lmax) 22 | :rtype: torch.Tensor 23 | """ 24 | from espnet.nets.pytorch_backend.nets_utils import pad_list 25 | 26 | _sos = ys_pad.new([sos]) 27 | _eos = ys_pad.new([eos]) 28 | ys = [y[y != ignore_id] for y in ys_pad] # parse padded ys 29 | ys_in = [torch.cat([_sos, y], dim=0) for y in ys] 30 | ys_out = [torch.cat([y, _eos], dim=0) for y in ys] 31 | return pad_list(ys_in, eos), pad_list(ys_out, ignore_id) 32 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/transformer/layer_norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright 2019 Shigeki Karita 5 | # Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) 6 | 7 | """Layer normalization module.""" 8 | 9 | import torch 10 | 11 | 12 | class LayerNorm(torch.nn.LayerNorm): 13 | """Layer normalization module. 14 | 15 | :param int nout: output dim size 16 | :param int dim: dimension to be normalized 17 | """ 18 | 19 | def __init__(self, nout, dim=-1): 20 | """Construct an LayerNorm object.""" 21 | super(LayerNorm, self).__init__(nout, eps=1e-12) 22 | self.dim = dim 23 | 24 | def forward(self, x): 25 | """Apply layer normalization. 26 | 27 | :param torch.Tensor x: input tensor 28 | :return: layer normalized tensor 29 | :rtype torch.Tensor 30 | """ 31 | if self.dim == -1: 32 | return super(LayerNorm, self).forward(x) 33 | return super(LayerNorm, self).forward(x.transpose(1, -1)).transpose(1, -1) 34 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/pytorch_backend/transformer/repeat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright 2019 Shigeki Karita 5 | # Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) 6 | 7 | """Repeat the same layer definition.""" 8 | 9 | import torch 10 | 11 | 12 | class MultiSequential(torch.nn.Sequential): 13 | """Multi-input multi-output torch.nn.Sequential.""" 14 | 15 | def forward(self, *args): 16 | """Repeat.""" 17 | for m in self: 18 | args = m(*args) 19 | return args 20 | 21 | 22 | def repeat(N, fn): 23 | """Repeat module N times. 24 | 25 | :param int N: repeat time 26 | :param function fn: function to generate module 27 | :return: repeated modules 28 | :rtype: MultiSequential 29 | """ 30 | return MultiSequential(*[fn() for _ in range(N)]) 31 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/nets/scorers/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/optimizer/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/optimizer/parser.py: -------------------------------------------------------------------------------- 1 | """Common optimizer default config for multiple backends.""" 2 | 3 | 4 | def sgd(parser): 5 | """Add arguments.""" 6 | parser.add_argument("--lr", type=float, default=1.0, help="Learning rate") 7 | parser.add_argument("--weight-decay", type=float, default=0.0, help="Weight decay") 8 | return parser 9 | 10 | 11 | def adam(parser): 12 | """Add arguments.""" 13 | parser.add_argument("--lr", type=float, default=1e-3, help="Learning rate") 14 | parser.add_argument("--beta1", type=float, default=0.9, help="Beta1") 15 | parser.add_argument("--beta2", type=float, default=0.999, help="Beta2") 16 | parser.add_argument("--weight-decay", type=float, default=0.0, help="Weight decay") 17 | return parser 18 | 19 | 20 | def adadelta(parser): 21 | """Add arguments.""" 22 | parser.add_argument("--rho", type=float, default=0.95, help="Rho") 23 | parser.add_argument("--eps", type=float, default=1e-8, help="Eps") 24 | parser.add_argument("--weight-decay", type=float, default=0.0, help="Weight decay") 25 | return parser 26 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/scheduler/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/scheduler/chainer.py: -------------------------------------------------------------------------------- 1 | """Chainer optimizer schdulers.""" 2 | 3 | from typing import List 4 | 5 | from chainer.optimizer import Optimizer 6 | 7 | from espnet.scheduler.scheduler import SchedulerInterface 8 | 9 | 10 | class ChainerScheduler: 11 | """Chainer optimizer scheduler.""" 12 | 13 | def __init__(self, schedulers: List[SchedulerInterface], optimizer: Optimizer): 14 | """Initialize class.""" 15 | self.schedulers = schedulers 16 | self.optimizer = optimizer 17 | self.init_values = dict() 18 | for s in self.schedulers: 19 | self.init_values[s.key] = getattr(self.optimizer, s.key) 20 | 21 | def step(self, n_iter: int): 22 | """Update optimizer by scheduling.""" 23 | for s in self.schedulers: 24 | new_val = self.init_values[s.key] * s.scale(n_iter) 25 | setattr(self.optimizer, s.key, new_val) 26 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/scheduler/pytorch.py: -------------------------------------------------------------------------------- 1 | """PyTorch optimizer schdulers.""" 2 | 3 | from typing import List 4 | 5 | from torch.optim import Optimizer 6 | 7 | from espnet.scheduler.scheduler import SchedulerInterface 8 | 9 | 10 | class PyTorchScheduler: 11 | """PyTorch optimizer scheduler.""" 12 | 13 | def __init__(self, schedulers: List[SchedulerInterface], optimizer: Optimizer): 14 | """Initialize class.""" 15 | self.schedulers = schedulers 16 | self.optimizer = optimizer 17 | for s in self.schedulers: 18 | for group in optimizer.param_groups: 19 | group.setdefault("initial_" + s.key, group[s.key]) 20 | 21 | def step(self, n_iter: int): 22 | """Update optimizer by scheduling.""" 23 | for s in self.schedulers: 24 | for group in self.optimizer.param_groups: 25 | group[s.key] = group["initial_" + s.key] * s.scale(n_iter) 26 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/st/pytorch_backend/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/transform/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize main package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/transform/add_deltas.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def delta(feat, window): 5 | assert window > 0 6 | delta_feat = np.zeros_like(feat) 7 | for i in range(1, window + 1): 8 | delta_feat[:-i] += i * feat[i:] 9 | delta_feat[i:] += -i * feat[:-i] 10 | delta_feat[-i:] += i * feat[-1] 11 | delta_feat[:i] += -i * feat[0] 12 | delta_feat /= 2 * sum(i ** 2 for i in range(1, window + 1)) 13 | return delta_feat 14 | 15 | 16 | def add_deltas(x, window=2, order=2): 17 | feats = [x] 18 | for _ in range(order): 19 | feats.append(delta(feats[-1], window)) 20 | return np.concatenate(feats, axis=1) 21 | 22 | 23 | class AddDeltas(object): 24 | def __init__(self, window=2, order=2): 25 | self.window = window 26 | self.order = order 27 | 28 | def __repr__(self): 29 | return "{name}(window={window}, order={order}".format( 30 | name=self.__class__.__name__, window=self.window, order=self.order 31 | ) 32 | 33 | def __call__(self, x): 34 | return add_deltas(x, window=self.window, order=self.order) 35 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/transform/transform_interface.py: -------------------------------------------------------------------------------- 1 | # TODO(karita): add this to all the transform impl. 2 | class TransformInterface: 3 | """Transform Interface""" 4 | 5 | def __call__(self, x): 6 | raise NotImplementedError("__call__ method is not implemented") 7 | 8 | @classmethod 9 | def add_arguments(cls, parser): 10 | return parser 11 | 12 | def __repr__(self): 13 | return self.__class__.__name__ + "()" 14 | 15 | 16 | class Identity(TransformInterface): 17 | """Identity Function""" 18 | 19 | def __call__(self, x): 20 | return x 21 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/tts/pytorch_backend/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/utils/check_kwargs.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | 3 | 4 | def check_kwargs(func, kwargs, name=None): 5 | """check kwargs are valid for func 6 | 7 | If kwargs are invalid, raise TypeError as same as python default 8 | :param function func: function to be validated 9 | :param dict kwargs: keyword arguments for func 10 | :param str name: name used in TypeError (default is func name) 11 | """ 12 | try: 13 | params = inspect.signature(func).parameters 14 | except ValueError: 15 | return 16 | if name is None: 17 | name = func.__name__ 18 | for k in kwargs.keys(): 19 | if k not in params: 20 | raise TypeError(f"{name}() got an unexpected keyword argument '{k}'") 21 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/utils/dynamic_import.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | 4 | def dynamic_import(import_path, alias=dict()): 5 | """dynamic import module and class 6 | 7 | :param str import_path: syntax 'module_name:class_name' 8 | e.g., 'espnet.transform.add_deltas:AddDeltas' 9 | :param dict alias: shortcut for registered class 10 | :return: imported class 11 | """ 12 | if import_path not in alias and ":" not in import_path: 13 | raise ValueError( 14 | "import_path should be one of {} or " 15 | 'include ":", e.g. "espnet.transform.add_deltas:AddDeltas" : ' 16 | "{}".format(set(alias), import_path) 17 | ) 18 | if ":" not in import_path: 19 | import_path = alias[import_path] 20 | 21 | module_name, objname = import_path.split(":") 22 | m = importlib.import_module(module_name) 23 | return getattr(m, objname) 24 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/utils/training/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialize sub package.""" 2 | -------------------------------------------------------------------------------- /Speech_Recognition/espnet/utils/training/evaluator.py: -------------------------------------------------------------------------------- 1 | from chainer.training.extensions import Evaluator 2 | 3 | from espnet.utils.training.tensorboard_logger import TensorboardLogger 4 | 5 | 6 | class BaseEvaluator(Evaluator): 7 | """Base Evaluator in ESPnet""" 8 | 9 | def __call__(self, trainer=None): 10 | ret = super().__call__(trainer) 11 | try: 12 | if trainer is not None: 13 | # force tensorboard to report evaluation log 14 | tb_logger = trainer.get_extension(TensorboardLogger.default_name) 15 | tb_logger(trainer) 16 | except ValueError: 17 | pass 18 | return ret 19 | -------------------------------------------------------------------------------- /Speech_Recognition/setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | 4 | [tool:pytest] 5 | addopts = --verbose --durations=0 --cov=espnet 6 | testpaths = test 7 | 8 | [flake8] 9 | ignore = H102,W504,H238 10 | # 120 is a workaround, 79 is good 11 | max-line-length = 120 12 | -------------------------------------------------------------------------------- /Speech_Recognition/tools/installers/install_chainer_ctc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | if [ $# != 0 ]; then 5 | echo "Usage: $0" 6 | exit 1; 7 | fi 8 | 9 | rm -rf chainer_ctc 10 | git clone https://github.com/jheymann85/chainer_ctc.git 11 | python3 -m pip install cython 12 | ( 13 | set -euo pipefail 14 | cd chainer_ctc && chmod +x install_warp-ctc.sh && ./install_warp-ctc.sh && python3 -m pip install . 15 | ) 16 | -------------------------------------------------------------------------------- /Speech_Recognition/tools/installers/install_kenlm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | if [ $# != 0 ]; then 5 | echo "Usage: $0" 6 | exit 1; 7 | fi 8 | 9 | rm -rf kenlm 10 | git clone https://github.com/kpu/kenlm.git 11 | ( 12 | set -euo pipefail 13 | cd kenlm 14 | 15 | mkdir build 16 | ( 17 | set -euo pipefail 18 | cd build && cmake .. && make 19 | ) 20 | ( 21 | set -euo pipefail 22 | python3 setup.py install 23 | ) 24 | ) 25 | -------------------------------------------------------------------------------- /Speech_Recognition/tools/installers/install_mwerSegmenter.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | 5 | if [ $# != 0 ]; then 6 | echo "Usage: $0" 7 | exit 1; 8 | fi 9 | 10 | wget --tries=3 https://www-i6.informatik.rwth-aachen.de/web/Software/mwerSegmenter.tar.gz 11 | tar zxvf mwerSegmenter.tar.gz 12 | rm mwerSegmenter.tar.gz 13 | -------------------------------------------------------------------------------- /Speech_Recognition/tools/installers/install_nkf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | 5 | if [ $# != 0 ]; then 6 | echo "Usage: $0" 7 | exit 1; 8 | fi 9 | 10 | 11 | rm -rf nkf 12 | mkdir -p nkf 13 | ( 14 | set -euo pipefail 15 | cd nkf 16 | wget --tries=3 https://ja.osdn.net/dl/nkf/nkf-2.1.4.tar.gz 17 | tar zxvf nkf-2.1.4.tar.gz 18 | ( 19 | set -euo pipefail 20 | cd nkf-2.1.4 21 | make prefix=. 22 | ) 23 | ) 24 | -------------------------------------------------------------------------------- /Speech_Recognition/tools/installers/install_pesq.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | 5 | if [ $# != 0 ]; then 6 | echo "Usage: $0" 7 | exit 1; 8 | fi 9 | 10 | if [ ! -e PESQ.zip ]; then 11 | wget --tries=3 'http://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-P.862-200511-I!Amd2!SOFT-ZST-E&type=items' -O PESQ.zip 12 | fi 13 | if [ ! -e PESQ ]; then 14 | mkdir -p PESQ_P.862.2 15 | unzip PESQ.zip -d PESQ_P.862.2 16 | unzip "PESQ_P.862.2/Software/P862_annex_A_2005_CD wav final.zip" -d PESQ_P.862.2 17 | rm -rf PESQ 18 | ln -s PESQ_P.862.2 PESQ 19 | fi 20 | 21 | ( 22 | set -euo pipefail 23 | cd PESQ/P862_annex_A_2005_CD/source 24 | gcc ./*.c -lm -o PESQ 25 | ) 26 | -------------------------------------------------------------------------------- /Speech_Recognition/tools/installers/install_sctk.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | 5 | if [ $# != 0 ]; then 6 | echo "Usage: $0" 7 | exit 1; 8 | fi 9 | 10 | 11 | if [ ! -e sctk-2.4.10-20151007-1312Z.tar.bz2 ]; then 12 | wget -T 10 https://github.com/espnet/kaldi-bin/releases/download/v0.0.2/sctk-2.4.10-20151007-1312Z.tar.bz2 || \ 13 | wget -T 10 -t 3 ftp://jaguar.ncsl.nist.gov/pub/sctk-2.4.10-20151007-1312Z.tar.bz2|| \ 14 | wget --no-check-certificate -T 10 http://www.openslr.org/resources/4/sctk-2.4.10-20151007-1312Z.tar.bz2 15 | fi 16 | 17 | if [ ! -e sctk ]; then 18 | tar xojf sctk-2.4.10-20151007-1312Z.tar.bz2 || \ 19 | tar --exclude '*NONE*html' -xvojf sctk-2.4.10-20151007-1312Z.tar.bz2 20 | rm -rf sctk && ln -s sctk-2.4.10 sctk 21 | fi 22 | 23 | ( 24 | set -euo pipefail 25 | cd sctk 26 | make config 27 | touch .configured 28 | make all && make install && make doc 29 | ) 30 | -------------------------------------------------------------------------------- /Speech_Recognition/tools/installers/install_sph2pipe.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | 5 | if [ $# != 0 ]; then 6 | echo "Usage: $0" 7 | exit 1; 8 | fi 9 | 10 | 11 | if [ ! -e sph2pipe_v2.5.tar.gz ]; then 12 | wget -T 10 https://github.com/espnet/kaldi-bin/releases/download/v0.0.2/sph2pipe_v2.5.tar.gz || \ 13 | wget -T 10 -t 3 http://www.openslr.org/resources/3/sph2pipe_v2.5.tar.gz || \ 14 | wget --no-check-certificate -T 10 https://sourceforge.net/projects/kaldi/files/sph2pipe_v2.5.tar.gz 15 | fi 16 | 17 | if [ ! -e sph2pipe_v2.5 ]; then 18 | tar --no-same-owner -xzf sph2pipe_v2.5.tar.gz 19 | fi 20 | 21 | ( 22 | set -euo pipefail 23 | cd sph2pipe_v2.5 24 | gcc -o sph2pipe *.c -lm 25 | ) 26 | -------------------------------------------------------------------------------- /Speech_Recognition/tools/installers/install_warp-transducer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | 5 | if [ $# != 0 ]; then 6 | echo "Usage: $0" 7 | exit 1; 8 | fi 9 | 10 | # TODO(kamo): Consider clang case 11 | # Note: Requires gcc>=4.9.2 to build extensions with pytorch>=1.0 12 | if python3 -c 'import torch as t;assert t.__version__[0] == "1"' &> /dev/null; then \ 13 | python3 -c "from distutils.version import LooseVersion as V;assert V('$(gcc -dumpversion)') >= V('4.9.2'), 'Requires gcc>=4.9.2'"; \ 14 | fi 15 | 16 | rm -rf warp-transducer 17 | git clone https://github.com/HawkAaron/warp-transducer.git 18 | 19 | ( 20 | set -euo pipefail 21 | cd warp-transducer 22 | 23 | mkdir build 24 | ( 25 | set -euo pipefail 26 | cd build && cmake .. && make 27 | ) 28 | 29 | ( 30 | set -euo pipefail 31 | cd pytorch_binding && python3 setup.py install 32 | ) 33 | ) 34 | -------------------------------------------------------------------------------- /Speech_Recognition/utils/feats2npy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import argparse 5 | from kaldiio import ReadHelper 6 | import numpy as np 7 | import os 8 | from os.path import join 9 | import sys 10 | 11 | 12 | def get_parser(): 13 | parser = argparse.ArgumentParser( 14 | description='Convet kaldi-style features to numpy arrays', 15 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 16 | parser.add_argument('scp_file', type=str, help='scp file') 17 | parser.add_argument('out_dir', type=str, help='output directory') 18 | return parser 19 | 20 | 21 | if __name__ == "__main__": 22 | args = get_parser().parse_args(sys.argv[1:]) 23 | os.makedirs(args.out_dir, exist_ok=True) 24 | with ReadHelper(f"scp:{args.scp_file}") as f: 25 | for utt_id, arr in f: 26 | out_path = join(args.out_dir, f"{utt_id}-feats.npy") 27 | np.save(out_path, arr, allow_pickle=False) 28 | sys.exit(0) 29 | -------------------------------------------------------------------------------- /Speech_Recognition/utils/get_yaml.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | 4 | import yaml 5 | 6 | 7 | def get_parser(): 8 | parser = argparse.ArgumentParser( 9 | description='get a specified attribute from a YAML file', 10 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 11 | parser.add_argument('inyaml') 12 | parser.add_argument('attr', help='foo.bar will access yaml.load(inyaml)["foo"]["bar"]') 13 | return parser 14 | 15 | 16 | def main(): 17 | args = get_parser().parse_args() 18 | with open(args.inyaml, 'r') as f: 19 | indict = yaml.load(f, Loader=yaml.Loader) 20 | 21 | try: 22 | for attr in args.attr.split('.'): 23 | if attr.isdigit(): 24 | attr = int(attr) 25 | indict = indict[attr] 26 | print(indict) 27 | except KeyError: 28 | # print nothing 29 | # sys.exit(1) 30 | pass 31 | 32 | 33 | if __name__ == '__main__': 34 | main() 35 | -------------------------------------------------------------------------------- /Speech_Recognition/utils/spm_train: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the license found in the 6 | # https://github.com/pytorch/fairseq/blob/master/LICENSE 7 | import sys 8 | 9 | import sentencepiece as spm 10 | 11 | 12 | if __name__ == "__main__": 13 | spm.SentencePieceTrainer.Train(" ".join(sys.argv[1:])) 14 | --------------------------------------------------------------------------------