├── .gitattributes
├── LICENSE
├── Machine_Translation
├── .github
│ ├── stale.yml
│ └── workflows
│ │ ├── build.yml
│ │ └── build_wheels.yml
├── .gitignore
├── .gitmodules
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── docs
│ ├── Makefile
│ ├── _static
│ │ └── theme_overrides.css
│ ├── command_line_tools.rst
│ ├── conf.py
│ ├── criterions.rst
│ ├── data.rst
│ ├── docutils.conf
│ ├── fairseq.gif
│ ├── fairseq_logo.png
│ ├── getting_started.rst
│ ├── hydra_integration.md
│ ├── index.rst
│ ├── lr_scheduler.rst
│ ├── make.bat
│ ├── models.rst
│ ├── modules.rst
│ ├── optim.rst
│ ├── overview.rst
│ ├── requirements.txt
│ ├── tasks.rst
│ ├── tutorial_classifying_names.rst
│ └── tutorial_simple_lstm.rst
├── examples
│ ├── .gitignore
│ ├── __init__.py
│ ├── adaptive_span
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── adagrad_with_grad_clip.py
│ │ ├── adaptive_span_attention.py
│ │ ├── adaptive_span_loss.py
│ │ ├── adaptive_span_model.py
│ │ ├── adaptive_span_model_wrapper.py
│ │ └── truncated_bptt_lm_task.py
│ ├── backtranslation
│ │ ├── README.md
│ │ ├── deduplicate_lines.py
│ │ ├── extract_bt_data.py
│ │ ├── prepare-de-monolingual.sh
│ │ ├── prepare-wmt18en2de.sh
│ │ ├── sacrebleu.sh
│ │ └── tokenized_bleu.sh
│ ├── bart
│ │ ├── README.glue.md
│ │ ├── README.md
│ │ ├── README.summarization.md
│ │ └── summarize.py
│ ├── byte_level_bpe
│ │ ├── README.md
│ │ ├── get_bitext.py
│ │ ├── get_data.sh
│ │ └── gru_transformer.py
│ ├── camembert
│ │ └── README.md
│ ├── constrained_decoding
│ │ ├── README.md
│ │ ├── normalize.py
│ │ └── tok.py
│ ├── conv_seq2seq
│ │ └── README.md
│ ├── criss
│ │ ├── README.md
│ │ ├── download_and_preprocess_flores_test.sh
│ │ ├── download_and_preprocess_tatoeba.sh
│ │ ├── mining
│ │ │ ├── mine.py
│ │ │ └── mine_example.sh
│ │ ├── save_encoder.py
│ │ ├── sentence_retrieval
│ │ │ ├── encoder_analysis.py
│ │ │ └── sentence_retrieval_tatoeba.sh
│ │ └── unsupervised_mt
│ │ │ └── eval.sh
│ ├── cross_lingual_language_model
│ │ └── README.md
│ ├── discriminative_reranking_nmt
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── config
│ │ │ └── deen.yaml
│ │ ├── criterions
│ │ │ ├── __init__.py
│ │ │ └── discriminative_reranking_criterion.py
│ │ ├── drnmt_rerank.py
│ │ ├── models
│ │ │ ├── __init__.py
│ │ │ └── discriminative_reranking_model.py
│ │ ├── scripts
│ │ │ └── prep_data.py
│ │ └── tasks
│ │ │ ├── __init__.py
│ │ │ └── discriminative_reranking_task.py
│ ├── fast_noisy_channel
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── noisy_channel_beam_search.py
│ │ ├── noisy_channel_sequence_generator.py
│ │ └── noisy_channel_translation.py
│ ├── flores101
│ │ ├── README.md
│ │ └── flores_logo.png
│ ├── fully_sharded_data_parallel
│ │ └── README.md
│ ├── gottbert
│ │ └── README.md
│ ├── hubert
│ │ ├── README.md
│ │ ├── config
│ │ │ ├── decode
│ │ │ │ ├── ax_sweep
│ │ │ │ │ ├── ngram.yaml
│ │ │ │ │ └── transformer.yaml
│ │ │ │ ├── infer_fsqlm.yaml
│ │ │ │ ├── infer_kenlm.yaml
│ │ │ │ ├── infer_viterbi.yaml
│ │ │ │ └── run
│ │ │ │ │ ├── submitit_slurm.yaml
│ │ │ │ │ └── submitit_slurm_8gpu.yaml
│ │ │ ├── finetune
│ │ │ │ ├── base_10h.yaml
│ │ │ │ ├── ckpt
│ │ │ │ │ └── it1.yaml
│ │ │ │ ├── lm
│ │ │ │ │ └── ls_4gram.yaml
│ │ │ │ └── run
│ │ │ │ │ └── submitit_reg.yaml
│ │ │ └── pretrain
│ │ │ │ ├── data
│ │ │ │ ├── iter1.yaml
│ │ │ │ └── iter2.yaml
│ │ │ │ ├── hubert_base_librispeech.yaml
│ │ │ │ ├── hubert_large_librivox.yaml
│ │ │ │ ├── hubert_xlarge_librivox.yaml
│ │ │ │ └── run
│ │ │ │ └── submitit_reg.yaml
│ │ ├── measure_teacher_quality.py
│ │ ├── simple_kmeans
│ │ │ ├── README.md
│ │ │ ├── dump_hubert_feature.py
│ │ │ ├── dump_hubert_feature_s2t.py
│ │ │ ├── dump_km_label.py
│ │ │ ├── dump_mfcc_feature.py
│ │ │ ├── dump_w2v2_feature.py
│ │ │ ├── feature_utils.py
│ │ │ └── learn_kmeans.py
│ │ └── update_ckpt.py
│ ├── joint_alignment_translation
│ │ ├── README.md
│ │ └── prepare-wmt18en2de_no_norm_no_escape_no_agressive.sh
│ ├── language_model
│ │ ├── README.adaptive_inputs.md
│ │ ├── README.conv.md
│ │ ├── README.md
│ │ └── prepare-wikitext-103.sh
│ ├── laser
│ │ ├── README.md
│ │ └── laser_src
│ │ │ ├── __init__.py
│ │ │ ├── laser_lstm.py
│ │ │ ├── laser_task.py
│ │ │ ├── laser_transformer.py
│ │ │ └── multitask_data_utils.py
│ ├── latent_depth
│ │ ├── README.md
│ │ └── latent_depth_src
│ │ │ ├── __init__.py
│ │ │ ├── loss
│ │ │ ├── __init__.py
│ │ │ └── latent_depth.py
│ │ │ ├── models
│ │ │ ├── __init__.py
│ │ │ ├── latent_multilingual_transformer.py
│ │ │ └── latent_transformer.py
│ │ │ ├── modules
│ │ │ ├── __init__.py
│ │ │ └── latent_layers.py
│ │ │ └── multilingual_translation_latent_depth.py
│ ├── layerdrop
│ │ └── README.md
│ ├── linformer
│ │ ├── README.md
│ │ └── linformer_src
│ │ │ ├── __init__.py
│ │ │ ├── models
│ │ │ ├── __init__.py
│ │ │ └── linformer_roberta.py
│ │ │ └── modules
│ │ │ ├── __init__.py
│ │ │ ├── linformer_sentence_encoder.py
│ │ │ ├── linformer_sentence_encoder_layer.py
│ │ │ └── multihead_linear_attention.py
│ ├── m2m_100
│ │ ├── README.md
│ │ ├── install_dependecies.sh
│ │ ├── process_data
│ │ │ ├── clean_histogram.py
│ │ │ ├── dedup_data.py
│ │ │ └── remove_too_much_punc.py
│ │ ├── tok.sh
│ │ └── tokenizers
│ │ │ ├── README.md
│ │ │ ├── seg_ja.sh
│ │ │ ├── seg_ko.sh
│ │ │ ├── thirdparty
│ │ │ └── .gitignore
│ │ │ ├── tokenize_indic.py
│ │ │ ├── tokenize_thai.py
│ │ │ ├── tokenize_zh.py
│ │ │ └── tokenizer_ar.sh
│ ├── mbart
│ │ └── README.md
│ ├── megatron_11b
│ │ ├── README.md
│ │ └── detok.py
│ ├── multilingual
│ │ ├── ML50_langs.txt
│ │ ├── README.md
│ │ ├── data_scripts
│ │ │ ├── README.md
│ │ │ ├── binarize.py
│ │ │ ├── check_iswlt_test_data.py
│ │ │ ├── check_self_overlaps.py
│ │ │ ├── check_valid_test_overlaps.py
│ │ │ ├── dedup_all.py
│ │ │ ├── download_ML50_v1.sh
│ │ │ ├── download_af_xh.sh
│ │ │ ├── download_flores_data.sh
│ │ │ ├── download_iitb.sh
│ │ │ ├── download_iwslt_and_extract.sh
│ │ │ ├── download_lotus.sh
│ │ │ ├── download_ted_and_extract.py
│ │ │ ├── download_wat19_my.sh
│ │ │ ├── download_wmt19_and_before.py
│ │ │ ├── download_wmt20.sh
│ │ │ ├── preprocess_ML50_v1.sh
│ │ │ ├── remove_valid_test_in_train.py
│ │ │ ├── requirement.txt
│ │ │ └── utils
│ │ │ │ ├── dedup.py
│ │ │ │ ├── fasttext_multi_filter.py
│ │ │ │ └── strip_sgm.sh
│ │ ├── finetune_multilingual_model.sh
│ │ ├── multilingual_fairseq_gen.sh
│ │ └── train_multilingual_model.sh
│ ├── noisychannel
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── rerank.py
│ │ ├── rerank_generate.py
│ │ ├── rerank_options.py
│ │ ├── rerank_score_bw.py
│ │ ├── rerank_score_lm.py
│ │ ├── rerank_tune.py
│ │ └── rerank_utils.py
│ ├── nonautoregressive_translation
│ │ ├── README.md
│ │ └── scripts.md
│ ├── paraphraser
│ │ ├── README.md
│ │ └── paraphrase.py
│ ├── pay_less_attention_paper
│ │ └── README.md
│ ├── pointer_generator
│ │ ├── README.md
│ │ ├── README.xsum.md
│ │ ├── pointer_generator_src
│ │ │ ├── __init__.py
│ │ │ └── transformer_pg.py
│ │ ├── postprocess.py
│ │ └── preprocess.py
│ ├── quant_noise
│ │ ├── README.md
│ │ └── transformer_quantization_config.yaml
│ ├── roberta
│ │ ├── README.custom_classification.md
│ │ ├── README.glue.md
│ │ ├── README.md
│ │ ├── README.pretraining.md
│ │ ├── README.race.md
│ │ ├── commonsense_qa
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── commonsense_qa_task.py
│ │ │ └── download_cqa_data.sh
│ │ ├── config
│ │ │ ├── finetuning
│ │ │ │ ├── cola.yaml
│ │ │ │ ├── mnli.yaml
│ │ │ │ ├── mrpc.yaml
│ │ │ │ ├── qnli.yaml
│ │ │ │ ├── qqp.yaml
│ │ │ │ ├── rte.yaml
│ │ │ │ ├── sst_2.yaml
│ │ │ │ └── sts_b.yaml
│ │ │ └── pretraining
│ │ │ │ └── base.yaml
│ │ ├── multiprocessing_bpe_encoder.py
│ │ ├── preprocess_GLUE_tasks.sh
│ │ ├── preprocess_RACE.py
│ │ ├── preprocess_RACE.sh
│ │ └── wsc
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── wsc_criterion.py
│ │ │ ├── wsc_task.py
│ │ │ └── wsc_utils.py
│ ├── rxf
│ │ ├── README.md
│ │ ├── __init__.py
│ │ └── rxf_src
│ │ │ ├── __init__.py
│ │ │ ├── label_smoothed_cross_entropy_r3f.py
│ │ │ └── sentence_prediction_r3f.py
│ ├── scaling_nmt
│ │ └── README.md
│ ├── simultaneous_translation
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── docs
│ │ │ ├── ende-mma.md
│ │ │ └── enja-waitk.md
│ │ ├── eval
│ │ │ └── agents
│ │ │ │ └── simul_t2t_enja.py
│ │ ├── models
│ │ │ ├── __init__.py
│ │ │ ├── convtransformer_simul_trans.py
│ │ │ └── transformer_monotonic_attention.py
│ │ ├── modules
│ │ │ ├── __init__.py
│ │ │ ├── fixed_pre_decision.py
│ │ │ ├── monotonic_multihead_attention.py
│ │ │ └── monotonic_transformer_layer.py
│ │ ├── tests
│ │ │ └── test_text_models.py
│ │ └── utils
│ │ │ ├── __init__.py
│ │ │ ├── data_utils.py
│ │ │ ├── functions.py
│ │ │ ├── latency.py
│ │ │ ├── monotonic_attention.py
│ │ │ └── p_choose_strategy.py
│ ├── speech_recognition
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── criterions
│ │ │ ├── ASG_loss.py
│ │ │ ├── __init__.py
│ │ │ └── cross_entropy_acc.py
│ │ ├── data
│ │ │ ├── __init__.py
│ │ │ ├── asr_dataset.py
│ │ │ ├── collaters.py
│ │ │ ├── data_utils.py
│ │ │ └── replabels.py
│ │ ├── datasets
│ │ │ ├── asr_prep_json.py
│ │ │ └── prepare-librispeech.sh
│ │ ├── infer.py
│ │ ├── kaldi
│ │ │ ├── __init__.py
│ │ │ ├── add-self-loop-simple.cc
│ │ │ ├── config
│ │ │ │ └── kaldi_initializer.yaml
│ │ │ ├── kaldi_decoder.py
│ │ │ └── kaldi_initializer.py
│ │ ├── models
│ │ │ ├── __init__.py
│ │ │ ├── vggtransformer.py
│ │ │ └── w2l_conv_glu_enc.py
│ │ ├── new
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── conf
│ │ │ │ ├── hydra
│ │ │ │ │ └── sweeper
│ │ │ │ │ │ └── ax.yaml
│ │ │ │ └── infer.yaml
│ │ │ ├── decoders
│ │ │ │ ├── __init__.py
│ │ │ │ ├── base_decoder.py
│ │ │ │ ├── decoder.py
│ │ │ │ ├── decoder_config.py
│ │ │ │ ├── flashlight_decoder.py
│ │ │ │ └── viterbi_decoder.py
│ │ │ └── infer.py
│ │ ├── tasks
│ │ │ ├── __init__.py
│ │ │ └── speech_recognition.py
│ │ ├── utils
│ │ │ └── wer_utils.py
│ │ └── w2l_decoder.py
│ ├── speech_text_joint_to_text
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── configs
│ │ │ └── mustc_noise.list
│ │ ├── criterions
│ │ │ ├── __init__.py
│ │ │ └── text_guide_cross_entropy_acc.py
│ │ ├── docs
│ │ │ ├── ende-mustc.md
│ │ │ └── iwslt2021.md
│ │ ├── models
│ │ │ ├── __init__.py
│ │ │ ├── s2t_dualinputtransformer.py
│ │ │ └── s2t_dualinputxmtransformer.py
│ │ ├── scripts
│ │ │ └── g2p_encode.py
│ │ └── tasks
│ │ │ ├── __init__.py
│ │ │ └── speech_text_joint.py
│ ├── speech_to_text
│ │ ├── README.md
│ │ ├── data_utils.py
│ │ ├── docs
│ │ │ ├── covost_example.md
│ │ │ ├── librispeech_example.md
│ │ │ ├── mtedx_example.md
│ │ │ ├── mustc_example.md
│ │ │ └── simulst_mustc_example.md
│ │ ├── prep_covost_data.py
│ │ ├── prep_librispeech_data.py
│ │ ├── prep_mtedx_data.py
│ │ ├── prep_mustc_data.py
│ │ ├── seg_mustc_data.py
│ │ └── simultaneous_translation
│ │ │ └── agents
│ │ │ └── fairseq_simul_st_agent.py
│ ├── stories
│ │ └── README.md
│ ├── translation
│ │ ├── README.md
│ │ ├── prepare-iwslt14.sh
│ │ ├── prepare-iwslt17-multilingual.sh
│ │ ├── prepare-wmt14en2de.sh
│ │ └── prepare-wmt14en2fr.sh
│ ├── translation_moe
│ │ ├── README.md
│ │ ├── score.py
│ │ └── translation_moe_src
│ │ │ ├── __init__.py
│ │ │ ├── logsumexp_moe.py
│ │ │ ├── mean_pool_gating_network.py
│ │ │ └── translation_moe.py
│ ├── truncated_bptt
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── transformer_xl_model.py
│ │ └── truncated_bptt_lm_task.py
│ ├── unsupervised_quality_estimation
│ │ ├── README.md
│ │ ├── aggregate_scores.py
│ │ ├── meteor.py
│ │ └── repeat_lines.py
│ ├── wav2vec
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── config
│ │ │ ├── finetuning
│ │ │ │ ├── base_100h.yaml
│ │ │ │ ├── base_10h.yaml
│ │ │ │ ├── base_10m.yaml
│ │ │ │ ├── base_1h.yaml
│ │ │ │ ├── base_960h.yaml
│ │ │ │ ├── vox_100h.yaml
│ │ │ │ ├── vox_10h.yaml
│ │ │ │ ├── vox_10m.yaml
│ │ │ │ ├── vox_1h.yaml
│ │ │ │ └── vox_960h.yaml
│ │ │ └── pretraining
│ │ │ │ ├── wav2vec2_base_librispeech.yaml
│ │ │ │ ├── wav2vec2_large_librivox.yaml
│ │ │ │ ├── wav2vec2_large_librivox_tpu-pod.yaml
│ │ │ │ └── wav2vec2_large_librivox_tpu.yaml
│ │ ├── libri_labels.py
│ │ ├── scripts
│ │ │ └── binarize_manifest.sh
│ │ ├── unsupervised
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config
│ │ │ │ ├── finetuning
│ │ │ │ │ └── w2v_finetune.yaml
│ │ │ │ ├── gan
│ │ │ │ │ └── w2vu.yaml
│ │ │ │ ├── generate
│ │ │ │ │ └── viterbi.yaml
│ │ │ │ ├── timit_matched
│ │ │ │ │ ├── test.uid
│ │ │ │ │ ├── train.uid
│ │ │ │ │ ├── train_text.uid
│ │ │ │ │ └── valid.uid
│ │ │ │ └── timit_unmatched
│ │ │ │ │ ├── test.uid
│ │ │ │ │ ├── train.uid
│ │ │ │ │ ├── train_text.uid
│ │ │ │ │ └── valid.uid
│ │ │ ├── data
│ │ │ │ ├── __init__.py
│ │ │ │ ├── extracted_features_dataset.py
│ │ │ │ └── random_input_dataset.py
│ │ │ ├── kaldi_self_train
│ │ │ │ ├── README.md
│ │ │ │ └── st
│ │ │ │ │ ├── cmd.sh
│ │ │ │ │ ├── decode_phone.sh
│ │ │ │ │ ├── decode_word_step1.sh
│ │ │ │ │ ├── decode_word_step2.sh
│ │ │ │ │ ├── local
│ │ │ │ │ ├── copy_aligned_text.py
│ │ │ │ │ ├── decode.sh
│ │ │ │ │ ├── prepare_data_from_w2v.py
│ │ │ │ │ ├── prepare_lang.sh
│ │ │ │ │ ├── prepare_lang_word.sh
│ │ │ │ │ ├── prepare_lm.sh
│ │ │ │ │ ├── score.sh
│ │ │ │ │ ├── show_wer.sh
│ │ │ │ │ ├── train_subset_lgbeam.sh
│ │ │ │ │ ├── unsup_select.py
│ │ │ │ │ ├── unsup_select_decode.sh
│ │ │ │ │ └── unsup_select_decode_word.sh
│ │ │ │ │ ├── path.sh
│ │ │ │ │ ├── steps
│ │ │ │ │ ├── steps_gan
│ │ │ │ │ ├── train_deltas.sh
│ │ │ │ │ ├── train_lda_mllt.sh
│ │ │ │ │ └── train_sat.sh
│ │ │ │ │ ├── train.sh
│ │ │ │ │ └── utils
│ │ │ ├── models
│ │ │ │ ├── __init__.py
│ │ │ │ └── wav2vec_u.py
│ │ │ ├── scripts
│ │ │ │ ├── apply_pca.py
│ │ │ │ ├── copy_labels.py
│ │ │ │ ├── filter_lexicon.py
│ │ │ │ ├── filter_tsv.py
│ │ │ │ ├── g2p_wrd_to_phn.py
│ │ │ │ ├── ltr_to_wrd.py
│ │ │ │ ├── mean_pool.py
│ │ │ │ ├── merge_clusters.py
│ │ │ │ ├── normalize_and_filter_text.py
│ │ │ │ ├── normalize_text.py
│ │ │ │ ├── pca.py
│ │ │ │ ├── phonemize_with_sil.py
│ │ │ │ ├── prepare_audio.sh
│ │ │ │ ├── prepare_text.sh
│ │ │ │ ├── prepare_timit.sh
│ │ │ │ ├── remove_silence.py
│ │ │ │ ├── vads.py
│ │ │ │ ├── wav2vec_apply_cluster_faiss.py
│ │ │ │ ├── wav2vec_cluster_faiss.py
│ │ │ │ ├── wav2vec_extract_features.py
│ │ │ │ ├── wer.py
│ │ │ │ └── wrd_to_ltr.py
│ │ │ ├── tasks
│ │ │ │ ├── __init__.py
│ │ │ │ └── unpaired_audio_text.py
│ │ │ └── w2vu_generate.py
│ │ ├── vq-wav2vec_featurize.py
│ │ ├── wav2vec_featurize.py
│ │ └── wav2vec_manifest.py
│ ├── wmt19
│ │ └── README.md
│ ├── wmt20
│ │ └── README.md
│ └── xlmr
│ │ └── README.md
├── fairseq
│ ├── __init__.py
│ ├── benchmark
│ │ ├── __init__.py
│ │ ├── dummy_dataset.py
│ │ ├── dummy_lm.py
│ │ ├── dummy_masked_lm.py
│ │ ├── dummy_model.py
│ │ └── dummy_mt.py
│ ├── binarizer.py
│ ├── checkpoint_utils.py
│ ├── clib
│ │ ├── cuda
│ │ │ ├── ngram_repeat_block_cuda.cpp
│ │ │ └── ngram_repeat_block_cuda_kernel.cu
│ │ ├── libbase
│ │ │ └── balanced_assignment.cpp
│ │ ├── libbleu
│ │ │ ├── libbleu.cpp
│ │ │ └── module.cpp
│ │ ├── libnat
│ │ │ └── edit_dist.cpp
│ │ └── libnat_cuda
│ │ │ ├── binding.cpp
│ │ │ ├── edit_dist.cu
│ │ │ └── edit_dist.h
│ ├── config
│ │ ├── __init__.py
│ │ ├── config.yaml
│ │ └── model
│ │ │ ├── transformer_lm
│ │ │ ├── transformer_lm_baevski_gbw.yaml
│ │ │ ├── transformer_lm_baevski_wiki103.yaml
│ │ │ ├── transformer_lm_big.yaml
│ │ │ ├── transformer_lm_gbw.yaml
│ │ │ ├── transformer_lm_gpt.yaml
│ │ │ ├── transformer_lm_gpt2_big.yaml
│ │ │ ├── transformer_lm_gpt2_medium.yaml
│ │ │ ├── transformer_lm_gpt2_small.yaml
│ │ │ └── transformer_lm_wiki103.yaml
│ │ │ ├── wav2vec
│ │ │ └── vq_wav2vec_gumbel.yaml
│ │ │ └── wav2vec2
│ │ │ ├── wav2vec2_base.yaml
│ │ │ └── wav2vec2_large.yaml
│ ├── criterions
│ │ ├── __init__.py
│ │ ├── adaptive_loss.py
│ │ ├── additional_loss.py
│ │ ├── composite_loss.py
│ │ ├── cross_entropy.py
│ │ ├── ctc.py
│ │ ├── fairseq_criterion.py
│ │ ├── hubert_criterion.py
│ │ ├── label_smoothed_cross_entropy.py
│ │ ├── label_smoothed_cross_entropy_latency_augmented.py
│ │ ├── label_smoothed_cross_entropy_with_alignment.py
│ │ ├── legacy_masked_lm.py
│ │ ├── masked_lm.py
│ │ ├── model_criterion.py
│ │ ├── nat_loss.py
│ │ ├── sentence_prediction.py
│ │ ├── sentence_ranking.py
│ │ └── wav2vec_criterion.py
│ ├── data
│ │ ├── __init__.py
│ │ ├── add_target_dataset.py
│ │ ├── append_token_dataset.py
│ │ ├── audio
│ │ │ ├── __init__.py
│ │ │ ├── audio_utils.py
│ │ │ ├── feature_transforms
│ │ │ │ ├── __init__.py
│ │ │ │ ├── global_cmvn.py
│ │ │ │ ├── specaugment.py
│ │ │ │ └── utterance_cmvn.py
│ │ │ ├── hubert_dataset.py
│ │ │ ├── multi_modality_dataset.py
│ │ │ ├── raw_audio_dataset.py
│ │ │ ├── speech_to_text_dataset.py
│ │ │ └── speech_to_text_joint_dataset.py
│ │ ├── backtranslation_dataset.py
│ │ ├── base_wrapper_dataset.py
│ │ ├── bucket_pad_length_dataset.py
│ │ ├── colorize_dataset.py
│ │ ├── concat_dataset.py
│ │ ├── concat_sentences_dataset.py
│ │ ├── data_utils.py
│ │ ├── data_utils_fast.pyx
│ │ ├── denoising_dataset.py
│ │ ├── dictionary.py
│ │ ├── encoders
│ │ │ ├── __init__.py
│ │ │ ├── byte_bpe.py
│ │ │ ├── byte_utils.py
│ │ │ ├── bytes.py
│ │ │ ├── characters.py
│ │ │ ├── fastbpe.py
│ │ │ ├── gpt2_bpe.py
│ │ │ ├── gpt2_bpe_utils.py
│ │ │ ├── hf_bert_bpe.py
│ │ │ ├── hf_byte_bpe.py
│ │ │ ├── moses_tokenizer.py
│ │ │ ├── nltk_tokenizer.py
│ │ │ ├── sentencepiece_bpe.py
│ │ │ ├── space_tokenizer.py
│ │ │ ├── subword_nmt_bpe.py
│ │ │ └── utils.py
│ │ ├── fairseq_dataset.py
│ │ ├── fasta_dataset.py
│ │ ├── id_dataset.py
│ │ ├── indexed_dataset.py
│ │ ├── iterators.py
│ │ ├── language_pair_dataset.py
│ │ ├── legacy
│ │ │ ├── __init__.py
│ │ │ ├── block_pair_dataset.py
│ │ │ ├── masked_lm_dataset.py
│ │ │ └── masked_lm_dictionary.py
│ │ ├── list_dataset.py
│ │ ├── lm_context_window_dataset.py
│ │ ├── lru_cache_dataset.py
│ │ ├── mask_tokens_dataset.py
│ │ ├── monolingual_dataset.py
│ │ ├── multi_corpus_dataset.py
│ │ ├── multi_corpus_sampled_dataset.py
│ │ ├── multilingual
│ │ │ ├── __init__.py
│ │ │ ├── multilingual_data_manager.py
│ │ │ ├── multilingual_utils.py
│ │ │ ├── sampled_multi_dataset.py
│ │ │ ├── sampled_multi_epoch_dataset.py
│ │ │ └── sampling_method.py
│ │ ├── nested_dictionary_dataset.py
│ │ ├── noising.py
│ │ ├── num_samples_dataset.py
│ │ ├── numel_dataset.py
│ │ ├── offset_tokens_dataset.py
│ │ ├── pad_dataset.py
│ │ ├── plasma_utils.py
│ │ ├── prepend_dataset.py
│ │ ├── prepend_token_dataset.py
│ │ ├── raw_label_dataset.py
│ │ ├── replace_dataset.py
│ │ ├── resampling_dataset.py
│ │ ├── roll_dataset.py
│ │ ├── round_robin_zip_datasets.py
│ │ ├── shorten_dataset.py
│ │ ├── sort_dataset.py
│ │ ├── strip_token_dataset.py
│ │ ├── subsample_dataset.py
│ │ ├── text_compressor.py
│ │ ├── token_block_dataset.py
│ │ ├── token_block_utils_fast.pyx
│ │ ├── transform_eos_dataset.py
│ │ └── transform_eos_lang_pair_dataset.py
│ ├── dataclass
│ │ ├── __init__.py
│ │ ├── configs.py
│ │ ├── constants.py
│ │ ├── initialize.py
│ │ └── utils.py
│ ├── distributed
│ │ ├── __init__.py
│ │ ├── distributed_timeout_wrapper.py
│ │ ├── fully_sharded_data_parallel.py
│ │ ├── legacy_distributed_data_parallel.py
│ │ ├── module_proxy_wrapper.py
│ │ ├── tpu_distributed_data_parallel.py
│ │ └── utils.py
│ ├── file_chunker_utils.py
│ ├── file_io.py
│ ├── file_utils.py
│ ├── hub_utils.py
│ ├── incremental_decoding_utils.py
│ ├── iterative_refinement_generator.py
│ ├── logging
│ │ ├── __init__.py
│ │ ├── meters.py
│ │ ├── metrics.py
│ │ └── progress_bar.py
│ ├── model_parallel
│ │ ├── __init__.py
│ │ ├── criterions
│ │ │ ├── __init__.py
│ │ │ └── vocab_parallel_cross_entropy.py
│ │ ├── megatron_trainer.py
│ │ ├── models
│ │ │ ├── __init__.py
│ │ │ ├── pipeline_parallel_transformer
│ │ │ │ ├── __init__.py
│ │ │ │ ├── layers.py
│ │ │ │ └── model.py
│ │ │ ├── roberta
│ │ │ │ ├── __init__.py
│ │ │ │ └── model.py
│ │ │ ├── transformer.py
│ │ │ └── transformer_lm.py
│ │ └── modules
│ │ │ ├── __init__.py
│ │ │ ├── multihead_attention.py
│ │ │ └── transformer_layer.py
│ ├── models
│ │ ├── __init__.py
│ │ ├── bart
│ │ │ ├── __init__.py
│ │ │ ├── hub_interface.py
│ │ │ └── model.py
│ │ ├── composite_encoder.py
│ │ ├── distributed_fairseq_model.py
│ │ ├── fairseq_decoder.py
│ │ ├── fairseq_encoder.py
│ │ ├── fairseq_incremental_decoder.py
│ │ ├── fairseq_model.py
│ │ ├── fconv.py
│ │ ├── fconv_lm.py
│ │ ├── fconv_self_att.py
│ │ ├── hubert
│ │ │ ├── __init__.py
│ │ │ ├── hubert.py
│ │ │ └── hubert_asr.py
│ │ ├── huggingface
│ │ │ ├── __init__.py
│ │ │ └── hf_gpt2.py
│ │ ├── lightconv.py
│ │ ├── lightconv_lm.py
│ │ ├── lstm.py
│ │ ├── lstm_lm.py
│ │ ├── masked_lm.py
│ │ ├── model_utils.py
│ │ ├── multilingual_transformer.py
│ │ ├── nat
│ │ │ ├── __init__.py
│ │ │ ├── cmlm_transformer.py
│ │ │ ├── fairseq_nat_model.py
│ │ │ ├── insertion_transformer.py
│ │ │ ├── iterative_nonautoregressive_transformer.py
│ │ │ ├── levenshtein_transformer.py
│ │ │ ├── levenshtein_utils.py
│ │ │ ├── nat_crf_transformer.py
│ │ │ ├── nonautoregressive_ensembles.py
│ │ │ └── nonautoregressive_transformer.py
│ │ ├── roberta
│ │ │ ├── __init__.py
│ │ │ ├── alignment_utils.py
│ │ │ ├── enc_dec.py
│ │ │ ├── hub_interface.py
│ │ │ ├── model.py
│ │ │ ├── model_camembert.py
│ │ │ ├── model_gottbert.py
│ │ │ └── model_xlmr.py
│ │ ├── speech_to_text
│ │ │ ├── __init__.py
│ │ │ ├── berard.py
│ │ │ ├── convtransformer.py
│ │ │ ├── modules
│ │ │ │ ├── augmented_memory_attention.py
│ │ │ │ └── emformer.py
│ │ │ ├── s2t_transformer.py
│ │ │ ├── utils.py
│ │ │ └── xm_transformer.py
│ │ ├── transformer.py
│ │ ├── transformer
│ │ │ ├── __init__.py
│ │ │ ├── transformer_base.py
│ │ │ ├── transformer_config.py
│ │ │ ├── transformer_decoder.py
│ │ │ ├── transformer_encoder.py
│ │ │ └── transformer_legacy.py
│ │ ├── transformer_align.py
│ │ ├── transformer_from_pretrained_xlm.py
│ │ ├── transformer_lm.py
│ │ └── wav2vec
│ │ │ ├── __init__.py
│ │ │ ├── wav2vec.py
│ │ │ ├── wav2vec2.py
│ │ │ └── wav2vec2_asr.py
│ ├── modules
│ │ ├── __init__.py
│ │ ├── adaptive_input.py
│ │ ├── adaptive_softmax.py
│ │ ├── base_layer.py
│ │ ├── beamable_mm.py
│ │ ├── character_token_embedder.py
│ │ ├── checkpoint_activations.py
│ │ ├── conv_tbc.py
│ │ ├── cross_entropy.py
│ │ ├── cuda_utils.cu
│ │ ├── downsampled_multihead_attention.py
│ │ ├── dynamic_convolution.py
│ │ ├── dynamic_crf_layer.py
│ │ ├── dynamicconv_layer
│ │ │ ├── __init__.py
│ │ │ ├── cuda_function_gen.py
│ │ │ ├── dynamicconv_cuda.cpp
│ │ │ ├── dynamicconv_cuda.cuh
│ │ │ ├── dynamicconv_cuda_kernel.cu
│ │ │ ├── dynamicconv_layer.py
│ │ │ ├── dynamiconv_cpu.cpp
│ │ │ └── setup.py
│ │ ├── fairseq_dropout.py
│ │ ├── fp32_group_norm.py
│ │ ├── gelu.py
│ │ ├── grad_multiply.py
│ │ ├── gumbel_vector_quantizer.py
│ │ ├── kmeans_attention.py
│ │ ├── kmeans_vector_quantizer.py
│ │ ├── layer_drop.py
│ │ ├── layer_norm.py
│ │ ├── learned_positional_embedding.py
│ │ ├── lightconv_layer
│ │ │ ├── __init__.py
│ │ │ ├── cuda_function_gen.py
│ │ │ ├── lightconv_cuda.cpp
│ │ │ ├── lightconv_cuda.cuh
│ │ │ ├── lightconv_cuda_kernel.cu
│ │ │ ├── lightconv_layer.py
│ │ │ └── setup.py
│ │ ├── lightweight_convolution.py
│ │ ├── linearized_convolution.py
│ │ ├── memory_component.py
│ │ ├── multihead_attention.py
│ │ ├── positional_embedding.py
│ │ ├── quant_noise.py
│ │ ├── quantization
│ │ │ ├── __init__.py
│ │ │ ├── pq
│ │ │ │ ├── __init__.py
│ │ │ │ ├── em.py
│ │ │ │ ├── modules
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── qconv.py
│ │ │ │ │ ├── qemb.py
│ │ │ │ │ └── qlinear.py
│ │ │ │ ├── pq.py
│ │ │ │ └── utils.py
│ │ │ ├── quantization_options.py
│ │ │ └── scalar
│ │ │ │ ├── __init__.py
│ │ │ │ ├── modules
│ │ │ │ ├── __init__.py
│ │ │ │ ├── qact.py
│ │ │ │ ├── qconv.py
│ │ │ │ ├── qemb.py
│ │ │ │ └── qlinear.py
│ │ │ │ ├── ops.py
│ │ │ │ └── utils.py
│ │ ├── same_pad.py
│ │ ├── scalar_bias.py
│ │ ├── sinusoidal_positional_embedding.py
│ │ ├── sparse_multihead_attention.py
│ │ ├── sparse_transformer_sentence_encoder.py
│ │ ├── sparse_transformer_sentence_encoder_layer.py
│ │ ├── transformer_layer.py
│ │ ├── transformer_sentence_encoder.py
│ │ ├── transformer_sentence_encoder_layer.py
│ │ ├── transpose_last.py
│ │ ├── unfold.py
│ │ └── vggblock.py
│ ├── nan_detector.py
│ ├── ngram_repeat_block.py
│ ├── optim
│ │ ├── __init__.py
│ │ ├── adadelta.py
│ │ ├── adafactor.py
│ │ ├── adagrad.py
│ │ ├── adam.py
│ │ ├── adamax.py
│ │ ├── amp_optimizer.py
│ │ ├── bmuf.py
│ │ ├── composite.py
│ │ ├── cpu_adam.py
│ │ ├── dynamic_loss_scaler.py
│ │ ├── fairseq_optimizer.py
│ │ ├── fp16_optimizer.py
│ │ ├── fused_adam.py
│ │ ├── fused_lamb.py
│ │ ├── lr_scheduler
│ │ │ ├── __init__.py
│ │ │ ├── cosine_lr_scheduler.py
│ │ │ ├── fairseq_lr_scheduler.py
│ │ │ ├── fixed_schedule.py
│ │ │ ├── inverse_square_root_schedule.py
│ │ │ ├── manual_lr_scheduler.py
│ │ │ ├── pass_through.py
│ │ │ ├── polynomial_decay_schedule.py
│ │ │ ├── reduce_lr_on_plateau.py
│ │ │ ├── tri_stage_lr_scheduler.py
│ │ │ └── triangular_lr_scheduler.py
│ │ ├── nag.py
│ │ ├── sgd.py
│ │ └── shard.py
│ ├── options.py
│ ├── pdb.py
│ ├── quantization_utils.py
│ ├── registry.py
│ ├── scoring
│ │ ├── __init__.py
│ │ ├── bleu.py
│ │ ├── chrf.py
│ │ ├── tokenizer.py
│ │ └── wer.py
│ ├── search.py
│ ├── sequence_generator.py
│ ├── sequence_scorer.py
│ ├── tasks
│ │ ├── __init__.py
│ │ ├── audio_finetuning.py
│ │ ├── audio_pretraining.py
│ │ ├── cross_lingual_lm.py
│ │ ├── denoising.py
│ │ ├── fairseq_task.py
│ │ ├── hubert_pretraining.py
│ │ ├── language_modeling.py
│ │ ├── legacy_masked_lm.py
│ │ ├── masked_lm.py
│ │ ├── multilingual_denoising.py
│ │ ├── multilingual_masked_lm.py
│ │ ├── multilingual_translation.py
│ │ ├── online_backtranslation.py
│ │ ├── semisupervised_translation.py
│ │ ├── sentence_prediction.py
│ │ ├── sentence_ranking.py
│ │ ├── simultaneous_translation.py
│ │ ├── speech_to_text.py
│ │ ├── translation.py
│ │ ├── translation_from_pretrained_bart.py
│ │ ├── translation_from_pretrained_xlm.py
│ │ ├── translation_lev.py
│ │ └── translation_multi_simple_epoch.py
│ ├── token_generation_constraints.py
│ ├── tokenizer.py
│ ├── trainer.py
│ ├── utils.py
│ └── version.txt
├── fairseq_cli
│ ├── __init__.py
│ ├── eval_lm.py
│ ├── generate.py
│ ├── hydra_train.py
│ ├── interactive.py
│ ├── preprocess.py
│ ├── score.py
│ ├── train.py
│ └── validate.py
├── hubconf.py
├── pyproject.toml
├── scripts
│ ├── __init__.py
│ ├── average_checkpoints.py
│ ├── build_sym_alignment.py
│ ├── compare_namespaces.py
│ ├── compound_split_bleu.sh
│ ├── constraints
│ │ ├── extract.py
│ │ └── validate.py
│ ├── convert_dictionary.lua
│ ├── convert_model.lua
│ ├── count_docs.py
│ ├── read_binarized.py
│ ├── rm_pt.py
│ ├── sacrebleu.sh
│ ├── shard_docs.py
│ ├── split_train_valid_docs.py
│ ├── spm_decode.py
│ ├── spm_encode.py
│ ├── spm_train.py
│ └── test_fsdp.sh
├── setup.py
├── tests
│ ├── __init__.py
│ ├── distributed
│ │ ├── __init__.py
│ │ ├── test_bmuf.py
│ │ ├── test_distributed_timeout_wrapper.py
│ │ ├── test_module_proxy_wrapper.py
│ │ ├── test_utils.py
│ │ └── utils.py
│ ├── gpu
│ │ ├── __init__.py
│ │ ├── test_binaries_gpu.py
│ │ └── transformer_quantization_config.yaml
│ ├── speech_recognition
│ │ ├── __init__.py
│ │ ├── asr_test_base.py
│ │ ├── test_collaters.py
│ │ ├── test_cross_entropy.py
│ │ ├── test_data_utils.py
│ │ └── test_vggtransformer.py
│ ├── test_activation_checkpointing.py
│ ├── test_amp_optimizer.py
│ ├── test_average_checkpoints.py
│ ├── test_backtranslation_dataset.py
│ ├── test_binaries.py
│ ├── test_character_token_embedder.py
│ ├── test_checkpoint_utils.py
│ ├── test_concat_dataset.py
│ ├── test_constraints.py
│ ├── test_convtbc.py
│ ├── test_data_utils.py
│ ├── test_dataclass_utils.py
│ ├── test_dataset.py
│ ├── test_dictionary.py
│ ├── test_export.py
│ ├── test_file_chunker_utils.py
│ ├── test_file_io.py
│ ├── test_fp16_optimizer.py
│ ├── test_inference_dropout.py
│ ├── test_iopath.py
│ ├── test_iterators.py
│ ├── test_label_smoothing.py
│ ├── test_lm_context_window.py
│ ├── test_lstm_jitable.py
│ ├── test_memory_efficient_fp16.py
│ ├── test_metrics.py
│ ├── test_multi_corpus_dataset.py
│ ├── test_multi_corpus_sampled_dataset.py
│ ├── test_multihead_attention.py
│ ├── test_noising.py
│ ├── test_online_backtranslation.py
│ ├── test_plasma_utils.py
│ ├── test_reproducibility.py
│ ├── test_resampling_dataset.py
│ ├── test_roberta.py
│ ├── test_sequence_generator.py
│ ├── test_sequence_scorer.py
│ ├── test_sparse_multihead_attention.py
│ ├── test_token_block_dataset.py
│ ├── test_train.py
│ ├── test_transformer.py
│ ├── test_utils.py
│ ├── test_valid_subset_checks.py
│ └── utils.py
└── train.py
├── README.md
└── Speech_Recognition
├── .gitignore
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── egs
└── aishell
│ └── asr
│ ├── RESULTS.md
│ ├── cmd.sh
│ ├── conf
│ ├── decode.yaml
│ ├── decode_memory_transformer.yaml
│ ├── decode_sync_transformer.yaml
│ ├── fbank.conf
│ ├── gpu.conf
│ ├── lm.yaml
│ ├── pitch.conf
│ ├── plot_compressive_transformer.yaml
│ ├── queue.conf
│ ├── slurm.conf
│ ├── specaug.yaml
│ ├── testing.yaml
│ ├── train.yaml
│ ├── train_memory_transformer.yaml
│ ├── train_synctransformer.yaml
│ ├── train_transducer.yaml
│ ├── train_transformer.yaml
│ └── tuning
│ │ ├── decode_pytorch_transformer.yaml
│ │ ├── decode_rnn.yaml
│ │ ├── train_pytorch_transformer.yaml
│ │ └── train_rnn.yaml
│ ├── dec_com.sh
│ ├── dec_sync.sh
│ ├── local
│ ├── aishell_data_prep.sh
│ ├── download_and_untar.sh
│ ├── make_voxceleb1.pl
│ ├── make_voxceleb1_v2.pl
│ ├── make_voxceleb2.pl
│ ├── nnet3
│ │ └── xvector
│ │ │ ├── prepare_feats_for_egs.sh
│ │ │ ├── run_xvector.sh
│ │ │ └── tuning
│ │ │ └── run_xvector_1a.sh
│ └── prepare_for_eer.py
│ ├── path.sh
│ ├── run.sh
│ ├── run_with_memory.sh
│ ├── sid
│ ├── compute_min_dcf.py
│ ├── compute_vad_decision.sh
│ ├── compute_vad_decision_gmm.sh
│ ├── extract_ivectors.sh
│ ├── extract_ivectors_dnn.sh
│ ├── gender_id.sh
│ ├── init_full_ubm_from_dnn.sh
│ ├── music_id.sh
│ ├── nnet2
│ │ ├── get_egs2.sh
│ │ ├── get_lda.sh
│ │ └── train_multisplice_accel2.sh
│ ├── nnet3
│ │ └── xvector
│ │ │ ├── allocate_egs.py
│ │ │ ├── extract_xvectors.sh
│ │ │ └── get_egs.sh
│ ├── train_diag_ubm.sh
│ ├── train_full_ubm.sh
│ ├── train_ivector_extractor.sh
│ └── train_ivector_extractor_dnn.sh
│ ├── steps
│ ├── align_basis_fmllr.sh
│ ├── align_basis_fmllr_lats.sh
│ ├── align_fmllr.sh
│ ├── align_fmllr_lats.sh
│ ├── align_lvtln.sh
│ ├── align_raw_fmllr.sh
│ ├── align_sgmm2.sh
│ ├── align_si.sh
│ ├── append_feats.sh
│ ├── best_path_weights.sh
│ ├── chain
│ │ ├── build_tree.sh
│ │ ├── build_tree_multiple_sources.sh
│ │ ├── e2e
│ │ │ ├── README.txt
│ │ │ ├── compute_biphone_stats.py
│ │ │ ├── get_egs_e2e.sh
│ │ │ ├── prepare_e2e.sh
│ │ │ ├── text_to_phones.py
│ │ │ └── train_e2e.py
│ │ ├── gen_topo.pl
│ │ ├── gen_topo.py
│ │ ├── gen_topo2.py
│ │ ├── gen_topo3.py
│ │ ├── gen_topo4.py
│ │ ├── gen_topo5.py
│ │ ├── gen_topo_orig.py
│ │ ├── get_egs.sh
│ │ ├── get_phone_post.sh
│ │ ├── make_weighted_den_fst.sh
│ │ ├── multilingual
│ │ │ └── combine_egs.sh
│ │ ├── train.py
│ │ └── train_tdnn.sh
│ ├── cleanup
│ │ ├── clean_and_segment_data.sh
│ │ ├── clean_and_segment_data_nnet3.sh
│ │ ├── combine_short_segments.py
│ │ ├── create_segments_from_ctm.pl
│ │ ├── debug_lexicon.sh
│ │ ├── decode_fmllr_segmentation.sh
│ │ ├── decode_segmentation.sh
│ │ ├── decode_segmentation_nnet3.sh
│ │ ├── find_bad_utts.sh
│ │ ├── find_bad_utts_nnet.sh
│ │ ├── internal
│ │ │ ├── align_ctm_ref.py
│ │ │ ├── compute_tf_idf.py
│ │ │ ├── ctm_to_text.pl
│ │ │ ├── get_ctm_edits.py
│ │ │ ├── get_non_scored_words.py
│ │ │ ├── get_pron_stats.py
│ │ │ ├── make_one_biased_lm.py
│ │ │ ├── modify_ctm_edits.py
│ │ │ ├── resolve_ctm_edits_overlaps.py
│ │ │ ├── retrieve_similar_docs.py
│ │ │ ├── segment_ctm_edits.py
│ │ │ ├── segment_ctm_edits_mild.py
│ │ │ ├── split_text_into_docs.pl
│ │ │ ├── stitch_documents.py
│ │ │ ├── taint_ctm_edits.py
│ │ │ └── tf_idf.py
│ │ ├── lattice_oracle_align.sh
│ │ ├── make_biased_lm_graphs.sh
│ │ ├── make_biased_lms.py
│ │ ├── make_segmentation_data_dir.sh
│ │ ├── make_segmentation_graph.sh
│ │ ├── make_utterance_fsts.pl
│ │ ├── make_utterance_graph.sh
│ │ ├── segment_long_utterances.sh
│ │ ├── segment_long_utterances_nnet3.sh
│ │ └── split_long_utterance.sh
│ ├── combine_ali_dirs.sh
│ ├── combine_lat_dirs.sh
│ ├── compare_alignments.sh
│ ├── compute_cmvn_stats.sh
│ ├── compute_vad_decision.sh
│ ├── conf
│ │ ├── append_eval_to_ctm.py
│ │ ├── append_prf_to_ctm.py
│ │ ├── apply_calibration.sh
│ │ ├── convert_ctm_to_tra.py
│ │ ├── get_ctm_conf.sh
│ │ ├── lattice_depth_per_frame.sh
│ │ ├── parse_arpa_unigrams.py
│ │ ├── prepare_calibration_data.py
│ │ ├── prepare_word_categories.py
│ │ └── train_calibration.sh
│ ├── copy_ali_dir.sh
│ ├── copy_lat_dir.sh
│ ├── data
│ │ ├── augment_data_dir.py
│ │ ├── data_dir_manipulation_lib.py
│ │ ├── make_musan.py
│ │ ├── make_musan.sh
│ │ └── reverberate_data_dir.py
│ ├── decode.sh
│ ├── decode_basis_fmllr.sh
│ ├── decode_biglm.sh
│ ├── decode_combine.sh
│ ├── decode_fmllr.sh
│ ├── decode_fmllr_extra.sh
│ ├── decode_fmmi.sh
│ ├── decode_fromlats.sh
│ ├── decode_lvtln.sh
│ ├── decode_nnet.sh
│ ├── decode_nolats.sh
│ ├── decode_raw_fmllr.sh
│ ├── decode_sgmm2.sh
│ ├── decode_sgmm2_fromlats.sh
│ ├── decode_sgmm2_rescore.sh
│ ├── decode_sgmm2_rescore_project.sh
│ ├── decode_si.sh
│ ├── decode_with_map.sh
│ ├── diagnostic
│ │ ├── analyze_alignments.sh
│ │ ├── analyze_lats.sh
│ │ ├── analyze_lattice_depth_stats.py
│ │ └── analyze_phone_length_stats.py
│ ├── dict
│ │ ├── apply_g2p.sh
│ │ ├── apply_g2p_phonetisaurus.sh
│ │ ├── apply_lexicon_edits.py
│ │ ├── get_pron_stats.py
│ │ ├── internal
│ │ │ ├── get_subsegments.py
│ │ │ ├── prune_pron_candidates.py
│ │ │ └── sum_arc_info.py
│ │ ├── learn_lexicon_bayesian.sh
│ │ ├── learn_lexicon_greedy.sh
│ │ ├── merge_learned_lexicons.py
│ │ ├── prons_to_lexicon.py
│ │ ├── prune_pron_candidates.py
│ │ ├── select_prons_bayesian.py
│ │ ├── select_prons_greedy.py
│ │ ├── train_g2p.sh
│ │ └── train_g2p_phonetisaurus.sh
│ ├── get_ctm.sh
│ ├── get_ctm_conf.sh
│ ├── get_ctm_conf_fast.sh
│ ├── get_ctm_fast.sh
│ ├── get_fmllr_basis.sh
│ ├── get_lexicon_probs.sh
│ ├── get_prons.sh
│ ├── get_train_ctm.sh
│ ├── info
│ │ ├── chain_dir_info.pl
│ │ ├── gmm_dir_info.pl
│ │ ├── nnet2_dir_info.pl
│ │ ├── nnet3_dir_info.pl
│ │ └── nnet3_disc_dir_info.pl
│ ├── libs
│ │ ├── __init__.py
│ │ ├── common.py
│ │ └── nnet3
│ │ │ ├── __init__.py
│ │ │ ├── report
│ │ │ ├── __init__.py
│ │ │ └── log_parse.py
│ │ │ ├── train
│ │ │ ├── __init__.py
│ │ │ ├── chain_objf
│ │ │ │ ├── __init__.py
│ │ │ │ └── acoustic_model.py
│ │ │ ├── common.py
│ │ │ ├── dropout_schedule.py
│ │ │ └── frame_level_objf
│ │ │ │ ├── __init__.py
│ │ │ │ ├── acoustic_model.py
│ │ │ │ ├── common.py
│ │ │ │ └── raw_model.py
│ │ │ └── xconfig
│ │ │ ├── __init__.py
│ │ │ ├── attention.py
│ │ │ ├── basic_layers.py
│ │ │ ├── composite_layers.py
│ │ │ ├── convolution.py
│ │ │ ├── gru.py
│ │ │ ├── layers.py
│ │ │ ├── lstm.py
│ │ │ ├── parser.py
│ │ │ ├── stats_layer.py
│ │ │ ├── trivial_layers.py
│ │ │ └── utils.py
│ ├── lmrescore.sh
│ ├── lmrescore_const_arpa.sh
│ ├── lmrescore_const_arpa_undeterminized.sh
│ ├── lmrescore_rnnlm_lat.sh
│ ├── make_denlats.sh
│ ├── make_denlats_sgmm2.sh
│ ├── make_fbank.sh
│ ├── make_fbank_pitch.sh
│ ├── make_index.sh
│ ├── make_mfcc.sh
│ ├── make_mfcc_pitch.sh
│ ├── make_mfcc_pitch_online.sh
│ ├── make_phone_graph.sh
│ ├── make_plp.sh
│ ├── make_plp_pitch.sh
│ ├── nnet
│ │ ├── align.sh
│ │ ├── decode.sh
│ │ ├── ivector
│ │ │ ├── extract_ivectors.sh
│ │ │ ├── train_diag_ubm.sh
│ │ │ └── train_ivector_extractor.sh
│ │ ├── make_bn_feats.sh
│ │ ├── make_denlats.sh
│ │ ├── make_fmllr_feats.sh
│ │ ├── make_fmmi_feats.sh
│ │ ├── make_priors.sh
│ │ ├── pretrain_dbn.sh
│ │ ├── train.sh
│ │ ├── train_mmi.sh
│ │ ├── train_mpe.sh
│ │ └── train_scheduler.sh
│ ├── nnet2
│ │ ├── adjust_priors.sh
│ │ ├── align.sh
│ │ ├── check_ivectors_compatible.sh
│ │ ├── convert_lda_to_raw.sh
│ │ ├── convert_nnet1_to_nnet2.sh
│ │ ├── create_appended_model.sh
│ │ ├── decode.sh
│ │ ├── dump_bottleneck_features.sh
│ │ ├── get_egs.sh
│ │ ├── get_egs2.sh
│ │ ├── get_egs_discriminative2.sh
│ │ ├── get_ivector_id.sh
│ │ ├── get_lda.sh
│ │ ├── get_lda_block.sh
│ │ ├── get_num_frames.sh
│ │ ├── get_perturbed_feats.sh
│ │ ├── make_denlats.sh
│ │ ├── make_multisplice_configs.py
│ │ ├── relabel_egs.sh
│ │ ├── relabel_egs2.sh
│ │ ├── remove_egs.sh
│ │ ├── retrain_fast.sh
│ │ ├── retrain_simple2.sh
│ │ ├── retrain_tanh.sh
│ │ ├── train_block.sh
│ │ ├── train_convnet_accel2.sh
│ │ ├── train_discriminative.sh
│ │ ├── train_discriminative2.sh
│ │ ├── train_discriminative_multilang2.sh
│ │ ├── train_more.sh
│ │ ├── train_more2.sh
│ │ ├── train_multilang2.sh
│ │ ├── train_multisplice_accel2.sh
│ │ ├── train_multisplice_ensemble.sh
│ │ ├── train_pnorm.sh
│ │ ├── train_pnorm_accel2.sh
│ │ ├── train_pnorm_bottleneck_fast.sh
│ │ ├── train_pnorm_ensemble.sh
│ │ ├── train_pnorm_fast.sh
│ │ ├── train_pnorm_multisplice.sh
│ │ ├── train_pnorm_multisplice2.sh
│ │ ├── train_pnorm_simple.sh
│ │ ├── train_pnorm_simple2.sh
│ │ ├── train_tanh.sh
│ │ ├── train_tanh_bottleneck.sh
│ │ ├── train_tanh_fast.sh
│ │ └── update_nnet.sh
│ ├── nnet3
│ │ ├── adjust_priors.sh
│ │ ├── align.sh
│ │ ├── align_lats.sh
│ │ ├── chain
│ │ │ ├── build_tree.sh
│ │ │ ├── build_tree_multiple_sources.sh
│ │ │ ├── e2e
│ │ │ │ ├── README.txt
│ │ │ │ ├── compute_biphone_stats.py
│ │ │ │ ├── get_egs_e2e.sh
│ │ │ │ ├── prepare_e2e.sh
│ │ │ │ ├── text_to_phones.py
│ │ │ │ └── train_e2e.py
│ │ │ ├── gen_topo.pl
│ │ │ ├── gen_topo.py
│ │ │ ├── gen_topo2.py
│ │ │ ├── gen_topo3.py
│ │ │ ├── gen_topo4.py
│ │ │ ├── gen_topo5.py
│ │ │ ├── gen_topo_orig.py
│ │ │ ├── get_egs.sh
│ │ │ ├── get_phone_post.sh
│ │ │ ├── make_weighted_den_fst.sh
│ │ │ ├── multilingual
│ │ │ │ └── combine_egs.sh
│ │ │ ├── train.py
│ │ │ └── train_tdnn.sh
│ │ ├── components.py
│ │ ├── compute_output.sh
│ │ ├── convert_nnet2_to_nnet3.py
│ │ ├── decode.sh
│ │ ├── decode_grammar.sh
│ │ ├── decode_looped.sh
│ │ ├── decode_score_fusion.sh
│ │ ├── decode_semisup.sh
│ │ ├── dot
│ │ │ ├── descriptor_parser.py
│ │ │ └── nnet3_to_dot.py
│ │ ├── get_degs.sh
│ │ ├── get_egs.sh
│ │ ├── get_egs_discriminative.sh
│ │ ├── get_egs_targets.sh
│ │ ├── get_saturation.pl
│ │ ├── get_successful_models.py
│ │ ├── lstm
│ │ │ ├── make_configs.py
│ │ │ └── train.sh
│ │ ├── make_bottleneck_features.sh
│ │ ├── make_denlats.sh
│ │ ├── make_tdnn_configs.py
│ │ ├── multilingual
│ │ │ ├── allocate_multilingual_examples.py
│ │ │ └── combine_egs.sh
│ │ ├── nnet3_to_dot.sh
│ │ ├── remove_egs.sh
│ │ ├── report
│ │ │ ├── convert_model.py
│ │ │ ├── generate_plots.py
│ │ │ └── summarize_compute_debug_timing.py
│ │ ├── tdnn
│ │ │ ├── make_configs.py
│ │ │ ├── train.sh
│ │ │ └── train_raw_nnet.sh
│ │ ├── train_discriminative.sh
│ │ ├── train_dnn.py
│ │ ├── train_raw_dnn.py
│ │ ├── train_raw_rnn.py
│ │ ├── train_rnn.py
│ │ ├── train_tdnn.sh
│ │ ├── xconfig_to_config.py
│ │ └── xconfig_to_configs.py
│ ├── online
│ │ ├── decode.sh
│ │ ├── nnet2
│ │ │ ├── align.sh
│ │ │ ├── copy_data_dir.sh
│ │ │ ├── copy_ivector_dir.sh
│ │ │ ├── decode.sh
│ │ │ ├── dump_nnet_activations.sh
│ │ │ ├── extract_ivectors.sh
│ │ │ ├── extract_ivectors_online.sh
│ │ │ ├── get_egs.sh
│ │ │ ├── get_egs2.sh
│ │ │ ├── get_egs_discriminative2.sh
│ │ │ ├── get_pca_transform.sh
│ │ │ ├── make_denlats.sh
│ │ │ ├── prepare_online_decoding.sh
│ │ │ ├── prepare_online_decoding_retrain.sh
│ │ │ ├── prepare_online_decoding_transfer.sh
│ │ │ ├── train_diag_ubm.sh
│ │ │ └── train_ivector_extractor.sh
│ │ ├── nnet3
│ │ │ ├── decode.sh
│ │ │ └── prepare_online_decoding.sh
│ │ └── prepare_online_decoding.sh
│ ├── oracle_wer.sh
│ ├── paste_feats.sh
│ ├── resegment_data.sh
│ ├── resegment_text.sh
│ ├── rnnlmrescore.sh
│ ├── score_kaldi.sh
│ ├── score_kaldi_compare.sh
│ ├── scoring
│ │ ├── score_kaldi_cer.sh
│ │ ├── score_kaldi_compare.sh
│ │ └── score_kaldi_wer.sh
│ ├── search_index.sh
│ ├── segmentation
│ │ ├── ali_to_targets.sh
│ │ ├── combine_targets_dirs.sh
│ │ ├── convert_targets_dir_to_whole_recording.sh
│ │ ├── convert_utt2spk_and_segments_to_rttm.py
│ │ ├── copy_targets_dir.sh
│ │ ├── decode_sad.sh
│ │ ├── detect_speech_activity.sh
│ │ ├── evaluate_segmentation.pl
│ │ ├── get_targets_for_out_of_segments.sh
│ │ ├── internal
│ │ │ ├── arc_info_to_targets.py
│ │ │ ├── find_oov_phone.py
│ │ │ ├── get_default_targets_for_out_of_segments.py
│ │ │ ├── get_transform_probs_mat.py
│ │ │ ├── merge_segment_targets_to_recording.py
│ │ │ ├── merge_targets.py
│ │ │ ├── prepare_sad_graph.py
│ │ │ ├── resample_targets.py
│ │ │ ├── sad_to_segments.py
│ │ │ └── verify_phones_list.py
│ │ ├── lats_to_targets.sh
│ │ ├── merge_targets_dirs.sh
│ │ ├── post_process_sad_to_segments.sh
│ │ ├── prepare_targets_gmm.sh
│ │ ├── resample_targets_dir.sh
│ │ └── validate_targets_dir.sh
│ ├── select_feats.sh
│ ├── shift_feats.sh
│ ├── subset_ali_dir.sh
│ ├── tandem
│ │ ├── align_fmllr.sh
│ │ ├── align_sgmm2.sh
│ │ ├── align_si.sh
│ │ ├── decode.sh
│ │ ├── decode_fmllr.sh
│ │ ├── decode_sgmm2.sh
│ │ ├── decode_si.sh
│ │ ├── make_denlats.sh
│ │ ├── make_denlats_sgmm2.sh
│ │ ├── mk_aslf_lda_mllt.sh
│ │ ├── mk_aslf_sgmm2.sh
│ │ ├── train_deltas.sh
│ │ ├── train_lda_mllt.sh
│ │ ├── train_mllt.sh
│ │ ├── train_mmi.sh
│ │ ├── train_mmi_sgmm2.sh
│ │ ├── train_mono.sh
│ │ ├── train_sat.sh
│ │ ├── train_sgmm2.sh
│ │ └── train_ubm.sh
│ ├── tfrnnlm
│ │ ├── check_py.py
│ │ ├── check_tensorflow_installed.sh
│ │ ├── lmrescore_rnnlm_lat.sh
│ │ ├── lmrescore_rnnlm_lat_pruned.sh
│ │ ├── lstm.py
│ │ ├── lstm_fast.py
│ │ ├── reader.py
│ │ └── vanilla_rnnlm.py
│ ├── train_deltas.sh
│ ├── train_diag_ubm.sh
│ ├── train_lda_mllt.sh
│ ├── train_lvtln.sh
│ ├── train_map.sh
│ ├── train_mmi.sh
│ ├── train_mmi_fmmi.sh
│ ├── train_mmi_fmmi_indirect.sh
│ ├── train_mmi_sgmm2.sh
│ ├── train_mono.sh
│ ├── train_mpe.sh
│ ├── train_nnet.sh
│ ├── train_quick.sh
│ ├── train_raw_sat.sh
│ ├── train_sat.sh
│ ├── train_sat_basis.sh
│ ├── train_segmenter.sh
│ ├── train_sgmm2.sh
│ ├── train_sgmm2_group.sh
│ ├── train_smbr.sh
│ ├── train_ubm.sh
│ └── word_align_lattices.sh
│ └── utils
│ ├── add_disambig.pl
│ ├── add_lex_disambig.pl
│ ├── analyze_segments.pl
│ ├── apply_map.pl
│ ├── best_wer.sh
│ ├── build_const_arpa_lm.sh
│ ├── combine_data.sh
│ ├── convert_ctm.pl
│ ├── convert_slf.pl
│ ├── convert_slf_parallel.sh
│ ├── copy_data_dir.sh
│ ├── create_data_link.pl
│ ├── create_split_dir.pl
│ ├── ctm
│ ├── convert_ctm.pl
│ ├── fix_ctm.sh
│ └── resolve_ctm_overlaps.py
│ ├── data
│ ├── combine_data.sh
│ ├── combine_short_segments.sh
│ ├── convert_data_dir_to_whole.sh
│ ├── copy_data_dir.sh
│ ├── extend_segment_times.py
│ ├── extract_wav_segments_data_dir.sh
│ ├── fix_data_dir.sh
│ ├── fix_subsegment_feats.pl
│ ├── get_frame_shift.sh
│ ├── get_num_frames.sh
│ ├── get_reco2dur.sh
│ ├── get_reco2utt_for_data.sh
│ ├── get_segments_for_data.sh
│ ├── get_uniform_subsegments.py
│ ├── get_utt2dur.sh
│ ├── get_utt2num_frames.sh
│ ├── internal
│ │ ├── choose_utts_to_combine.py
│ │ ├── combine_segments_to_recording.py
│ │ ├── modify_speaker_info.py
│ │ └── perturb_volume.py
│ ├── limit_feature_dim.sh
│ ├── modify_speaker_info.sh
│ ├── modify_speaker_info_to_recording.sh
│ ├── normalize_data_range.pl
│ ├── perturb_data_dir_speed.sh
│ ├── perturb_data_dir_speed_3way.sh
│ ├── perturb_data_dir_volume.sh
│ ├── perturb_speed_to_allowed_lengths.py
│ ├── remove_dup_utts.sh
│ ├── resample_data_dir.sh
│ ├── shift_and_combine_feats.sh
│ ├── shift_feats.sh
│ ├── split_data.sh
│ ├── subsegment_data_dir.sh
│ ├── subset_data_dir.sh
│ └── validate_data_dir.sh
│ ├── dict_dir_add_pronprobs.sh
│ ├── eps2disambig.pl
│ ├── filt.py
│ ├── filter_scp.pl
│ ├── filter_scps.pl
│ ├── find_arpa_oovs.pl
│ ├── fix_ctm.sh
│ ├── fix_data_dir.sh
│ ├── format_lm.sh
│ ├── format_lm_sri.sh
│ ├── gen_topo.pl
│ ├── int2sym.pl
│ ├── kwslist_post_process.pl
│ ├── lang
│ ├── add_lex_disambig.pl
│ ├── add_unigrams_arpa.pl
│ ├── adjust_unk_arpa.pl
│ ├── adjust_unk_graph.sh
│ ├── bpe
│ │ ├── add_final_optional_silence.sh
│ │ ├── apply_bpe.py
│ │ ├── bidi.py
│ │ ├── learn_bpe.py
│ │ ├── prepend_words.py
│ │ └── reverse.py
│ ├── check_g_properties.pl
│ ├── check_phones_compatible.sh
│ ├── compute_sentence_probs_arpa.py
│ ├── extend_lang.sh
│ ├── get_word_position_phone_map.pl
│ ├── grammar
│ │ ├── augment_phones_txt.py
│ │ └── augment_words_txt.py
│ ├── internal
│ │ ├── apply_unk_lm.sh
│ │ ├── arpa2fst_constrained.py
│ │ └── modify_unk_pron.py
│ ├── limit_arpa_unk_history.py
│ ├── make_kn_lm.py
│ ├── make_lexicon_fst.py
│ ├── make_lexicon_fst_silprob.py
│ ├── make_phone_bigram_lang.sh
│ ├── make_phone_lm.py
│ ├── make_position_dependent_subword_lexicon.py
│ ├── make_subword_lexicon_fst.py
│ ├── make_unk_lm.sh
│ ├── prepare_lang.sh
│ ├── validate_disambig_sym_file.pl
│ └── validate_lang.pl
│ ├── ln.pl
│ ├── make_absolute.sh
│ ├── make_lexicon_fst.pl
│ ├── make_lexicon_fst_silprob.pl
│ ├── make_unigram_grammar.pl
│ ├── map_arpa_lm.pl
│ ├── mkgraph.sh
│ ├── nnet-cpu
│ ├── make_nnet_config.pl
│ ├── make_nnet_config_block.pl
│ ├── make_nnet_config_preconditioned.pl
│ └── update_learning_rates.pl
│ ├── nnet
│ ├── gen_dct_mat.py
│ ├── gen_hamm_mat.py
│ ├── gen_splice.py
│ ├── make_blstm_proto.py
│ ├── make_cnn_proto.py
│ ├── make_lstm_proto.py
│ ├── make_nnet_proto.py
│ └── subset_data_tr_cv.sh
│ ├── parallel
│ ├── limit_num_gpus.sh
│ ├── pbs.pl
│ ├── queue.pl
│ ├── retry.pl
│ ├── run.pl
│ └── slurm.pl
│ ├── parse_options.sh
│ ├── pbs.pl
│ ├── perturb_data_dir_speed.sh
│ ├── pinyin_map.pl
│ ├── prepare_extended_lang.sh
│ ├── prepare_lang.sh
│ ├── prepare_online_nnet_dist_build.sh
│ ├── queue.pl
│ ├── remove_data_links.sh
│ ├── remove_oovs.pl
│ ├── retry.pl
│ ├── reverse_arpa.py
│ ├── rnnlm_compute_scores.sh
│ ├── run.pl
│ ├── s2eps.pl
│ ├── scoring
│ ├── wer_ops_details.pl
│ ├── wer_per_spk_details.pl
│ ├── wer_per_utt_details.pl
│ └── wer_report.pl
│ ├── segmentation.pl
│ ├── show_lattice.sh
│ ├── shuffle_list.pl
│ ├── slurm.pl
│ ├── spk2utt_to_utt2spk.pl
│ ├── split_data.sh
│ ├── split_scp.pl
│ ├── ssh.pl
│ ├── subset_data_dir.sh
│ ├── subset_data_dir_tr_cv.sh
│ ├── subset_scp.pl
│ ├── subword
│ ├── prepare_lang_subword.sh
│ └── prepare_subword_text.sh
│ ├── summarize_logs.pl
│ ├── summarize_warnings.pl
│ ├── sym2int.pl
│ ├── utt2spk_to_spk2utt.pl
│ ├── validate_data_dir.sh
│ ├── validate_dict_dir.pl
│ ├── validate_lang.pl
│ ├── validate_text.pl
│ └── write_kwslist.pl
├── espnet
├── __init__.py
├── asr
│ ├── __init__.py
│ ├── asr_mix_utils.py
│ ├── asr_utils.py
│ ├── chainer_backend
│ │ ├── __init__.py
│ │ └── asr.py
│ └── pytorch_backend
│ │ ├── __init__.py
│ │ ├── asr.py
│ │ ├── asr_init.py
│ │ ├── asr_mix.py
│ │ └── recog.py
├── bin
│ ├── __init__.py
│ ├── asr_enhance.py
│ ├── asr_recog.py
│ ├── asr_train.py
│ ├── lm_train.py
│ ├── mt_train.py
│ ├── mt_trans.py
│ ├── st_train.py
│ ├── st_trans.py
│ ├── tts_decode.py
│ └── tts_train.py
├── lm
│ ├── __init__.py
│ ├── chainer_backend
│ │ ├── __init__.py
│ │ ├── extlm.py
│ │ └── lm.py
│ ├── lm_utils.py
│ └── pytorch_backend
│ │ ├── __init__.py
│ │ ├── extlm.py
│ │ └── lm.py
├── mt
│ ├── __init__.py
│ ├── mt_utils.py
│ └── pytorch_backend
│ │ ├── __init__.py
│ │ └── mt.py
├── nets
│ ├── __init__.py
│ ├── asr_interface.py
│ ├── batch_beam_search.py
│ ├── beam_search.py
│ ├── chainer_backend
│ │ ├── __init__.py
│ │ ├── asr_interface.py
│ │ ├── ctc.py
│ │ ├── deterministic_embed_id.py
│ │ ├── e2e_asr.py
│ │ ├── e2e_asr_transformer.py
│ │ ├── nets_utils.py
│ │ ├── rnn
│ │ │ ├── __init__.py
│ │ │ ├── attentions.py
│ │ │ ├── decoders.py
│ │ │ ├── encoders.py
│ │ │ └── training.py
│ │ └── transformer
│ │ │ ├── __init__.py
│ │ │ ├── attention.py
│ │ │ ├── ctc.py
│ │ │ ├── decoder.py
│ │ │ ├── decoder_layer.py
│ │ │ ├── embedding.py
│ │ │ ├── encoder.py
│ │ │ ├── encoder_layer.py
│ │ │ ├── label_smoothing_loss.py
│ │ │ ├── layer_norm.py
│ │ │ ├── mask.py
│ │ │ ├── plot.py
│ │ │ ├── positionwise_feed_forward.py
│ │ │ ├── subsampling.py
│ │ │ └── training.py
│ ├── ctc_prefix_score.py
│ ├── e2e_asr_common.py
│ ├── e2e_mt_common.py
│ ├── lm_interface.py
│ ├── mt_interface.py
│ ├── pytorch_backend
│ │ ├── __init__.py
│ │ ├── compressive_transducer
│ │ │ ├── __init__.py
│ │ │ ├── encoder.py
│ │ │ ├── encoder_layer.py
│ │ │ ├── initializer.py
│ │ │ ├── loss.py
│ │ │ ├── rnn_decoders.py
│ │ │ ├── transformer_decoder.py
│ │ │ ├── transformer_decoder_layer.py
│ │ │ ├── utils.py
│ │ │ └── vgg.py
│ │ ├── ctc.py
│ │ ├── e2e_asr.py
│ │ ├── e2e_asr_compressive_transducer.py
│ │ ├── e2e_asr_mix.py
│ │ ├── e2e_asr_mulenc.py
│ │ ├── e2e_asr_sync_transducer.py
│ │ ├── e2e_asr_transducer.py
│ │ ├── e2e_asr_transformer.py
│ │ ├── e2e_mt.py
│ │ ├── e2e_mt_transformer.py
│ │ ├── e2e_st.py
│ │ ├── e2e_st_transformer.py
│ │ ├── e2e_tts_fastspeech.py
│ │ ├── e2e_tts_tacotron2.py
│ │ ├── e2e_tts_transformer.py
│ │ ├── fastspeech
│ │ │ ├── __init__.py
│ │ │ ├── duration_calculator.py
│ │ │ ├── duration_predictor.py
│ │ │ └── length_regulator.py
│ │ ├── frontends
│ │ │ ├── __init__.py
│ │ │ ├── beamformer.py
│ │ │ ├── dnn_beamformer.py
│ │ │ ├── dnn_wpe.py
│ │ │ ├── feature_transform.py
│ │ │ ├── frontend.py
│ │ │ └── mask_estimator.py
│ │ ├── initialization.py
│ │ ├── lm
│ │ │ ├── __init__.py
│ │ │ ├── default.py
│ │ │ ├── seq_rnn.py
│ │ │ └── transformer.py
│ │ ├── memory_component.py
│ │ ├── nets_utils.py
│ │ ├── rnn
│ │ │ ├── __init__.py
│ │ │ ├── attentions.py
│ │ │ ├── decoders.py
│ │ │ └── encoders.py
│ │ ├── streaming
│ │ │ ├── __init__.py
│ │ │ ├── segment.py
│ │ │ └── window.py
│ │ ├── sync_transducer
│ │ │ ├── __init__.py
│ │ │ ├── encoder.py
│ │ │ ├── encoder_layer.py
│ │ │ ├── initializer.py
│ │ │ ├── loss.py
│ │ │ ├── rnn_decoders.py
│ │ │ ├── transformer_decoder.py
│ │ │ ├── transformer_decoder_layer.py
│ │ │ ├── utils.py
│ │ │ └── vgg.py
│ │ ├── tacotron2
│ │ │ ├── __init__.py
│ │ │ ├── cbhg.py
│ │ │ ├── decoder.py
│ │ │ └── encoder.py
│ │ ├── transducer
│ │ │ ├── __init__.py
│ │ │ ├── initializer.py
│ │ │ ├── loss.py
│ │ │ ├── rnn_decoders.py
│ │ │ ├── transformer_decoder.py
│ │ │ ├── transformer_decoder_layer.py
│ │ │ ├── utils.py
│ │ │ └── vgg.py
│ │ ├── transformer
│ │ │ ├── __init__.py
│ │ │ ├── add_sos_eos.py
│ │ │ ├── additional_loss.py
│ │ │ ├── attention.py
│ │ │ ├── decoder.py
│ │ │ ├── decoder_layer.py
│ │ │ ├── embedding.py
│ │ │ ├── encoder.py
│ │ │ ├── encoder_layer.py
│ │ │ ├── initializer.py
│ │ │ ├── label_smoothing_loss.py
│ │ │ ├── layer_norm.py
│ │ │ ├── mask.py
│ │ │ ├── multi_layer_conv.py
│ │ │ ├── optimizer.py
│ │ │ ├── plot.py
│ │ │ ├── positionwise_feed_forward.py
│ │ │ ├── repeat.py
│ │ │ └── subsampling.py
│ │ └── wavenet.py
│ ├── scorer_interface.py
│ ├── scorers
│ │ ├── __init__.py
│ │ ├── ctc.py
│ │ └── length_bonus.py
│ ├── st_interface.py
│ └── tts_interface.py
├── optimizer
│ ├── __init__.py
│ ├── chainer.py
│ ├── factory.py
│ ├── parser.py
│ └── pytorch.py
├── scheduler
│ ├── __init__.py
│ ├── chainer.py
│ ├── pytorch.py
│ └── scheduler.py
├── st
│ └── pytorch_backend
│ │ ├── __init__.py
│ │ └── st.py
├── transform
│ ├── __init__.py
│ ├── add_deltas.py
│ ├── channel_selector.py
│ ├── cmvn.py
│ ├── functional.py
│ ├── perturb.py
│ ├── spec_augment.py
│ ├── spectrogram.py
│ ├── transform_interface.py
│ ├── transformation.py
│ └── wpe.py
├── tts
│ └── pytorch_backend
│ │ ├── __init__.py
│ │ └── tts.py
└── utils
│ ├── __init__.py
│ ├── check_kwargs.py
│ ├── cli_readers.py
│ ├── cli_utils.py
│ ├── cli_writers.py
│ ├── dataset.py
│ ├── deterministic_utils.py
│ ├── dynamic_import.py
│ ├── fill_missing_args.py
│ ├── io_utils.py
│ ├── spec_augment.py
│ └── training
│ ├── __init__.py
│ ├── batchfy.py
│ ├── evaluator.py
│ ├── iterators.py
│ ├── tensorboard_logger.py
│ └── train_utils.py
├── setup.cfg
├── setup.py
├── tools
├── Makefile
├── check_install.py
├── installers
│ ├── install_chainer_ctc.sh
│ ├── install_kenlm.sh
│ ├── install_mwerSegmenter.sh
│ ├── install_nkf.sh
│ ├── install_pesq.sh
│ ├── install_pyopenjtalk.sh
│ ├── install_sctk.sh
│ ├── install_sph2pipe.sh
│ ├── install_warp-ctc.sh
│ └── install_warp-transducer.sh
└── setup_anaconda.sh
└── utils
├── addjson.py
├── apply-cmvn.py
├── average_checkpoints.py
├── change_yaml.py
├── compute-cmvn-stats.py
├── compute-fbank-feats.py
├── compute-stft-feats.py
├── concatjson.py
├── convert_fbank.sh
├── convert_fbank_to_wav.py
├── copy-feats.py
├── data2json.sh
├── download_from_google_drive.sh
├── dump-pcm.py
├── dump.sh
├── dump_pcm.sh
├── eval-source-separation.py
├── eval_perm_free_error.py
├── eval_source_separation.sh
├── feat-to-shape.py
├── feat_to_shape.sh
├── feats2npy.py
├── filt.py
├── free-gpu.sh
├── generate_wav.sh
├── generate_wav_from_fbank.py
├── get_yaml.py
├── json2sctm.py
├── json2text.py
├── json2trn.py
├── json2trn_mt.py
├── json2trn_wo_dict.py
├── make_fbank.sh
├── make_stft.sh
├── merge_scp2json.py
├── mergejson.py
├── mix-mono-wav-scp.py
├── pack_model.sh
├── queue-freegpu.pl
├── recog_wav.sh
├── reduce_data_dir.sh
├── remove_longshortdata.sh
├── result2json.py
├── score_bleu.sh
├── score_sclite.sh
├── score_sclite_wo_dict.sh
├── scp2json.py
├── show_result.sh
├── splitjson.py
├── spm_decode
├── spm_encode
├── spm_train
├── stdout.pl
├── synth_wav.sh
├── text2token.py
├── text2vocabulary.py
├── trim_silence.py
├── trim_silence.sh
├── trn2ctm.py
└── trn2stm.py
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2021 kobekobe0410853
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/Machine_Translation/.github/workflows/build_wheels.yml:
--------------------------------------------------------------------------------
1 | name: build_wheels
2 |
3 | on:
4 | push:
5 | branches:
6 | - v[0-9]+.[0-9]+.[x0-9]+
7 | tags:
8 | - v*
9 |
10 | jobs:
11 | build_wheels:
12 | name: Build wheels on ${{ matrix.os }}
13 | runs-on: ${{ matrix.os }}
14 | strategy:
15 | matrix:
16 | os: [ubuntu-latest, macos-latest]
17 |
18 | steps:
19 | - uses: actions/checkout@v2
20 |
21 | - name: Install Python
22 | uses: actions/setup-python@v2
23 | with:
24 | python-version: '3.7'
25 |
26 | - name: Install cibuildwheel
27 | run: |
28 | python -m pip install cibuildwheel
29 |
30 | - name: Build wheels for CPython
31 | run: |
32 | python -m cibuildwheel --output-dir dist
33 | env:
34 | CIBW_BUILD: "cp36-*64 cp37-*64 cp38-*64"
35 | CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
36 | CIBW_BEFORE_BUILD: git submodule update --init --recursive && pip install .
37 |
38 | - uses: actions/upload-artifact@v2
39 | with:
40 | name: wheels
41 | path: ./dist/*.whl
42 |
--------------------------------------------------------------------------------
/Machine_Translation/.gitmodules:
--------------------------------------------------------------------------------
1 | [submodule "fairseq/model_parallel/megatron"]
2 | path = fairseq/model_parallel/megatron
3 | url = https://github.com/ngoyal2707/Megatron-LM
4 | branch = fairseq
5 |
--------------------------------------------------------------------------------
/Machine_Translation/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) Facebook, Inc. and its affiliates.
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/Machine_Translation/docs/Makefile:
--------------------------------------------------------------------------------
1 | # Minimal makefile for Sphinx documentation
2 | #
3 |
4 | # You can set these variables from the command line.
5 | SPHINXOPTS =
6 | SPHINXBUILD = python -msphinx
7 | SPHINXPROJ = fairseq
8 | SOURCEDIR = .
9 | BUILDDIR = _build
10 |
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 |
15 | .PHONY: help Makefile
16 |
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
--------------------------------------------------------------------------------
/Machine_Translation/docs/_static/theme_overrides.css:
--------------------------------------------------------------------------------
1 | .wy-table-responsive table td kbd {
2 | white-space: nowrap;
3 | }
4 | .wy-table-responsive table td {
5 | white-space: normal !important;
6 | }
7 | .wy-table-responsive {
8 | overflow: visible !important;
9 | }
10 |
--------------------------------------------------------------------------------
/Machine_Translation/docs/criterions.rst:
--------------------------------------------------------------------------------
1 | .. role:: hidden
2 | :class: hidden-section
3 |
4 | .. _Criterions:
5 |
6 | Criterions
7 | ==========
8 |
9 | Criterions compute the loss function given the model and batch, roughly::
10 |
11 | loss = criterion(model, batch)
12 |
13 | .. automodule:: fairseq.criterions
14 | :members:
15 |
16 | .. autoclass:: fairseq.criterions.FairseqCriterion
17 | :members:
18 | :undoc-members:
19 |
20 | .. autoclass:: fairseq.criterions.adaptive_loss.AdaptiveLoss
21 | :members:
22 | :undoc-members:
23 | .. autoclass:: fairseq.criterions.composite_loss.CompositeLoss
24 | :members:
25 | :undoc-members:
26 | .. autoclass:: fairseq.criterions.cross_entropy.CrossEntropyCriterion
27 | :members:
28 | :undoc-members:
29 | .. autoclass:: fairseq.criterions.label_smoothed_cross_entropy.LabelSmoothedCrossEntropyCriterion
30 | :members:
31 | :undoc-members:
32 |
--------------------------------------------------------------------------------
/Machine_Translation/docs/docutils.conf:
--------------------------------------------------------------------------------
1 | [writers]
2 | option-limit=0
3 |
--------------------------------------------------------------------------------
/Machine_Translation/docs/fairseq.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/docs/fairseq.gif
--------------------------------------------------------------------------------
/Machine_Translation/docs/fairseq_logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/docs/fairseq_logo.png
--------------------------------------------------------------------------------
/Machine_Translation/docs/lr_scheduler.rst:
--------------------------------------------------------------------------------
1 | .. role:: hidden
2 | :class: hidden-section
3 |
4 | .. _Learning Rate Schedulers:
5 |
6 | Learning Rate Schedulers
7 | ========================
8 |
9 | Learning Rate Schedulers update the learning rate over the course of training.
10 | Learning rates can be updated after each update via :func:`step_update` or at
11 | epoch boundaries via :func:`step`.
12 |
13 | .. automodule:: fairseq.optim.lr_scheduler
14 | :members:
15 |
16 | .. autoclass:: fairseq.optim.lr_scheduler.FairseqLRScheduler
17 | :members:
18 | :undoc-members:
19 |
20 | .. autoclass:: fairseq.optim.lr_scheduler.cosine_lr_scheduler.CosineSchedule
21 | :members:
22 | :undoc-members:
23 | .. autoclass:: fairseq.optim.lr_scheduler.fixed_schedule.FixedSchedule
24 | :members:
25 | :undoc-members:
26 | .. autoclass:: fairseq.optim.lr_scheduler.inverse_square_root_schedule.InverseSquareRootSchedule
27 | :members:
28 | :undoc-members:
29 | .. autoclass:: fairseq.optim.lr_scheduler.reduce_lr_on_plateau.ReduceLROnPlateau
30 | :members:
31 | :undoc-members:
32 | .. autoclass:: fairseq.optim.lr_scheduler.triangular_lr_scheduler.TriangularSchedule
33 | :members:
34 | :undoc-members:
35 |
--------------------------------------------------------------------------------
/Machine_Translation/docs/make.bat:
--------------------------------------------------------------------------------
1 | @ECHO OFF
2 |
3 | pushd %~dp0
4 |
5 | REM Command file for Sphinx documentation
6 |
7 | if "%SPHINXBUILD%" == "" (
8 | set SPHINXBUILD=python -msphinx
9 | )
10 | set SOURCEDIR=.
11 | set BUILDDIR=_build
12 | set SPHINXPROJ=fairseq
13 |
14 | if "%1" == "" goto help
15 |
16 | %SPHINXBUILD% >NUL 2>NUL
17 | if errorlevel 9009 (
18 | echo.
19 | echo.The Sphinx module was not found. Make sure you have Sphinx installed,
20 | echo.then set the SPHINXBUILD environment variable to point to the full
21 | echo.path of the 'sphinx-build' executable. Alternatively you may add the
22 | echo.Sphinx directory to PATH.
23 | echo.
24 | echo.If you don't have Sphinx installed, grab it from
25 | echo.http://sphinx-doc.org/
26 | exit /b 1
27 | )
28 |
29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
30 | goto end
31 |
32 | :help
33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
34 |
35 | :end
36 | popd
37 |
--------------------------------------------------------------------------------
/Machine_Translation/docs/modules.rst:
--------------------------------------------------------------------------------
1 | Modules
2 | =======
3 |
4 | Fairseq provides several stand-alone :class:`torch.nn.Module` classes that may
5 | be helpful when implementing a new :class:`~fairseq.models.BaseFairseqModel`.
6 |
7 | .. automodule:: fairseq.modules
8 | :members:
9 | :undoc-members:
10 |
--------------------------------------------------------------------------------
/Machine_Translation/docs/optim.rst:
--------------------------------------------------------------------------------
1 | .. role:: hidden
2 | :class: hidden-section
3 |
4 | .. _optimizers:
5 |
6 | Optimizers
7 | ==========
8 |
9 | Optimizers update the Model parameters based on the gradients.
10 |
11 | .. automodule:: fairseq.optim
12 | :members:
13 |
14 | .. autoclass:: fairseq.optim.FairseqOptimizer
15 | :members:
16 | :undoc-members:
17 |
18 | .. autoclass:: fairseq.optim.adadelta.Adadelta
19 | :members:
20 | :undoc-members:
21 | .. autoclass:: fairseq.optim.adagrad.Adagrad
22 | :members:
23 | :undoc-members:
24 | .. autoclass:: fairseq.optim.adafactor.FairseqAdafactor
25 | :members:
26 | :undoc-members:
27 | .. autoclass:: fairseq.optim.adam.FairseqAdam
28 | :members:
29 | :undoc-members:
30 | .. autoclass:: fairseq.optim.fp16_optimizer.FP16Optimizer
31 | :members:
32 | :undoc-members:
33 | .. autoclass:: fairseq.optim.nag.FairseqNAG
34 | :members:
35 | :undoc-members:
36 | .. autoclass:: fairseq.optim.sgd.SGD
37 | :members:
38 | :undoc-members:
39 |
--------------------------------------------------------------------------------
/Machine_Translation/docs/requirements.txt:
--------------------------------------------------------------------------------
1 | sphinx<2.0
2 | sphinx-argparse
3 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/.gitignore:
--------------------------------------------------------------------------------
1 | !*/*.sh
2 | !*/*.md
3 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | try:
7 | from fairseq.version import __version__ # noqa
8 | except ImportError:
9 | pass
10 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/adaptive_span/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import importlib
7 | import os
8 |
9 | # automatically import any Python files in the current directory
10 | cur_dir = os.path.dirname(__file__)
11 | for file in os.listdir(cur_dir):
12 | path = os.path.join(cur_dir, file)
13 | if (
14 | not file.startswith("_")
15 | and not file.startswith(".")
16 | and (file.endswith(".py") or os.path.isdir(path))
17 | ):
18 | mod_name = file[: file.find(".py")] if file.endswith(".py") else file
19 | module = importlib.import_module(__name__ + "." + mod_name)
20 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/adaptive_span/truncated_bptt_lm_task.py:
--------------------------------------------------------------------------------
1 | ../truncated_bptt/truncated_bptt_lm_task.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/backtranslation/sacrebleu.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | if [ $# -ne 5 ]; then
4 | echo "usage: $0 [dataset=wmt14/full] [langpair=en-de] [databin] [bpecode] [model]"
5 | exit
6 | fi
7 |
8 |
9 | DATASET=$1
10 | LANGPAIR=$2
11 | DATABIN=$3
12 | BPECODE=$4
13 | MODEL=$5
14 |
15 | SRCLANG=$(echo $LANGPAIR | cut -d '-' -f 1)
16 | TGTLANG=$(echo $LANGPAIR | cut -d '-' -f 2)
17 |
18 |
19 | BPEROOT=examples/backtranslation/subword-nmt/subword_nmt
20 | if [ ! -e $BPEROOT ]; then
21 | BPEROOT=subword-nmt/subword_nmt
22 | if [ ! -e $BPEROOT ]; then
23 | echo 'Cloning Subword NMT repository (for BPE pre-processing)...'
24 | git clone https://github.com/rsennrich/subword-nmt.git
25 | fi
26 | fi
27 |
28 |
29 | sacrebleu -t $DATASET -l $LANGPAIR --echo src \
30 | | sacremoses tokenize -a -l $SRCLANG -q \
31 | | python $BPEROOT/apply_bpe.py -c $BPECODE \
32 | | fairseq-interactive $DATABIN --path $MODEL \
33 | -s $SRCLANG -t $TGTLANG \
34 | --beam 5 --remove-bpe --buffer-size 1024 --max-tokens 8000 \
35 | | grep ^H- | cut -f 3- \
36 | | sacremoses detokenize -l $TGTLANG -q \
37 | | sacrebleu -t $DATASET -l $LANGPAIR
38 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/constrained_decoding/normalize.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | #
3 | # Copyright (c) Facebook, Inc. and its affiliates.
4 | #
5 | # This source code is licensed under the MIT license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 | import sys
9 |
10 | from sacremoses.normalize import MosesPunctNormalizer
11 |
12 |
13 | def main(args):
14 | normalizer = MosesPunctNormalizer(lang=args.lang, penn=args.penn)
15 | for line in sys.stdin:
16 | print(normalizer.normalize(line.rstrip()), flush=True)
17 |
18 |
19 | if __name__ == "__main__":
20 | import argparse
21 |
22 | parser = argparse.ArgumentParser()
23 | parser.add_argument("--lang", "-l", default="en")
24 | parser.add_argument("--penn", "-p", action="store_true")
25 | args = parser.parse_args()
26 |
27 | main(args)
28 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/constrained_decoding/tok.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | #
3 | # Copyright (c) Facebook, Inc. and its affiliates.
4 | #
5 | # This source code is licensed under the MIT license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 | import sys
9 |
10 | import sacremoses
11 |
12 |
13 | def main(args):
14 | """Tokenizes, preserving tabs"""
15 | mt = sacremoses.MosesTokenizer(lang=args.lang)
16 |
17 | def tok(s):
18 | return mt.tokenize(s, return_str=True)
19 |
20 | for line in sys.stdin:
21 | parts = list(map(tok, line.split("\t")))
22 | print(*parts, sep="\t", flush=True)
23 |
24 |
25 | if __name__ == "__main__":
26 | import argparse
27 |
28 | parser = argparse.ArgumentParser()
29 | parser.add_argument("--lang", "-l", default="en")
30 | parser.add_argument("--penn", "-p", action="store_true")
31 | parser.add_argument("--fields", "-f", help="fields to tokenize")
32 | args = parser.parse_args()
33 |
34 | main(args)
35 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/discriminative_reranking_nmt/__init__.py:
--------------------------------------------------------------------------------
1 | from . import criterions, models, tasks # noqa
2 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/discriminative_reranking_nmt/criterions/__init__.py:
--------------------------------------------------------------------------------
1 | from .discriminative_reranking_criterion import KLDivergenceRerankingCriterion
2 |
3 |
4 | __all__ = [
5 | "KLDivergenceRerankingCriterion",
6 | ]
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/discriminative_reranking_nmt/models/__init__.py:
--------------------------------------------------------------------------------
1 | from .discriminative_reranking_model import DiscriminativeNMTReranker
2 |
3 |
4 | __all__ = [
5 | "DiscriminativeNMTReranker",
6 | ]
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/discriminative_reranking_nmt/tasks/__init__.py:
--------------------------------------------------------------------------------
1 | from .discriminative_reranking_task import DiscriminativeRerankingNMTTask
2 |
3 |
4 | __all__ = [
5 | "DiscriminativeRerankingNMTTask",
6 | ]
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/fast_noisy_channel/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import noisy_channel_translation # noqa
7 | from . import noisy_channel_sequence_generator # noqa
8 | from . import noisy_channel_beam_search # noqa
9 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/flores101/flores_logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/flores101/flores_logo.png
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/decode/ax_sweep/ngram.yaml:
--------------------------------------------------------------------------------
1 | # @package _global_
2 |
3 | common_eval:
4 | results_path: ${decoding.exp_dir}/decode/${decoding.decoder.name}_ax/${dataset.gen_subset}
5 |
6 | hydra:
7 | sweeper:
8 | ax_config:
9 | max_trials: 60
10 | early_stop:
11 | minimize: true
12 | max_epochs_without_improvement: 10
13 | epsilon: 0.025
14 | experiment:
15 | name: ${dataset.gen_subset}
16 | objective_name: wer
17 | minimize: true
18 | parameter_constraints: null
19 | outcome_constraints: null
20 | status_quo: null
21 | client:
22 | verbose_logging: false
23 | random_seed: null
24 | params:
25 | decoding.decoder.lmweight:
26 | type: range
27 | bounds: [0.0, 8.0]
28 | decoding.decoder.wordscore:
29 | type: range
30 | bounds: [-5.0, 5.0]
31 | decoding.decoder.silweight:
32 | type: range
33 | bounds: [-10.0, 0.0]
34 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/decode/ax_sweep/transformer.yaml:
--------------------------------------------------------------------------------
1 | # @package _global_
2 |
3 | common_eval:
4 | results_path: ${decoding.exp_dir}/decode/${decoding.decoder.name}_ax/${dataset.gen_subset}
5 |
6 | hydra:
7 | sweeper:
8 | ax_config:
9 | max_trials: 60
10 | early_stop:
11 | minimize: true
12 | max_epochs_without_improvement: 10
13 | epsilon: 0.025
14 | experiment:
15 | name: ${dataset.gen_subset}
16 | objective_name: wer
17 | minimize: true
18 | parameter_constraints: null
19 | outcome_constraints: null
20 | status_quo: null
21 | client:
22 | verbose_logging: false
23 | random_seed: null
24 | params:
25 | decoding.decoder.lmweight:
26 | type: range
27 | bounds: [0.0, 4.0]
28 | decoding.decoder.wordscore:
29 | type: range
30 | bounds: [-5.0, 5.0]
31 | decoding.decoder.silweight:
32 | type: range
33 | bounds: [-8.0, 0.0]
34 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/decode/infer_fsqlm.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 |
3 | defaults:
4 | - model: null
5 |
6 | hydra:
7 | run:
8 | dir: ${common_eval.results_path}/beam${decoding.beam}_th${decoding.beamthreshold}_lmw${decoding.lmweight}_wrd${decoding.wordscore}_sil${decoding.silweight}
9 | sweep:
10 | dir: ${common_eval.results_path}
11 | subdir: beam${decoding.beam}_th${decoding.beamthreshold}_lmw${decoding.lmweight}_wrd${decoding.wordscore}_sil${decoding.silweight}
12 |
13 | task:
14 | _name: hubert_pretraining
15 | single_target: true
16 | fine_tuning: true
17 | data: ???
18 | normalize: ???
19 |
20 | decoding:
21 | type: fairseqlm
22 | lexicon: ???
23 | lmpath: ???
24 | beamthreshold: 25
25 | beam: 500
26 | lmweight: 2
27 | wordscore: -1
28 | silweight: 0
29 | unique_wer_file: true
30 | common_eval:
31 | results_path: ???
32 | path: ???
33 | post_process: letter
34 | dataset:
35 | max_tokens: 1100000
36 | gen_subset: ???
37 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/decode/infer_kenlm.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 |
3 | defaults:
4 | - model: null
5 |
6 | hydra:
7 | run:
8 | dir: ${common_eval.results_path}/beam${decoding.beam}_th${decoding.beamthreshold}_lmw${decoding.lmweight}_wrd${decoding.wordscore}_sil${decoding.silweight}
9 | sweep:
10 | dir: ${common_eval.results_path}
11 | subdir: beam${decoding.beam}_th${decoding.beamthreshold}_lmw${decoding.lmweight}_wrd${decoding.wordscore}_sil${decoding.silweight}
12 |
13 | task:
14 | _name: hubert_pretraining
15 | single_target: true
16 | fine_tuning: true
17 | data: ???
18 | normalize: ???
19 |
20 | decoding:
21 | type: kenlm
22 | lexicon: ???
23 | lmpath: ???
24 | beamthreshold: 100
25 | beam: 500
26 | lmweight: 2
27 | wordscore: -1
28 | silweight: 0
29 | unique_wer_file: true
30 | common_eval:
31 | results_path: ???
32 | path: ???
33 | post_process: letter
34 | dataset:
35 | max_tokens: 1100000
36 | gen_subset: ???
37 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/decode/infer_viterbi.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 |
3 | defaults:
4 | - model: null
5 |
6 | hydra:
7 | run:
8 | dir: ${common_eval.results_path}/viterbi
9 | sweep:
10 | dir: ${common_eval.results_path}
11 | subdir: viterbi
12 |
13 | task:
14 | _name: hubert_pretraining
15 | single_target: true
16 | fine_tuning: true
17 | data: ???
18 | normalize: ???
19 |
20 | decoding:
21 | type: viterbi
22 | unique_wer_file: true
23 | common_eval:
24 | results_path: ???
25 | path: ???
26 | post_process: letter
27 | dataset:
28 | max_tokens: 1100000
29 | gen_subset: ???
30 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/decode/run/submitit_slurm.yaml:
--------------------------------------------------------------------------------
1 | # @package _global_
2 | hydra:
3 | launcher:
4 | cpus_per_task: ${distributed_training.distributed_world_size}
5 | gpus_per_node: ${distributed_training.distributed_world_size}
6 | tasks_per_node: ${hydra.launcher.gpus_per_node}
7 | nodes: 1
8 | mem_gb: 200
9 | timeout_min: 4320
10 | max_num_timeout: 50
11 | name: ${hydra.job.config_name}
12 | submitit_folder: ${hydra.sweep.dir}/submitit
13 |
14 | distributed_training:
15 | distributed_world_size: 1
16 | distributed_no_spawn: true
17 | distributed_port: 29761
18 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/decode/run/submitit_slurm_8gpu.yaml:
--------------------------------------------------------------------------------
1 | # @package _global_
2 | hydra:
3 | launcher:
4 | cpus_per_task: ${distributed_training.distributed_world_size}
5 | gpus_per_node: ${distributed_training.distributed_world_size}
6 | tasks_per_node: ${hydra.launcher.gpus_per_node}
7 | nodes: 1
8 | mem_gb: 200
9 | timeout_min: 4320
10 | max_num_timeout: 50
11 | name: ${hydra.job.config_name}
12 | submitit_folder: ${hydra.sweep.dir}/submitit
13 |
14 | distributed_training:
15 | distributed_world_size: 8
16 | distributed_no_spawn: true
17 | distributed_port: 29761
18 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/finetune/ckpt/it1.yaml:
--------------------------------------------------------------------------------
1 | # @package _global_
2 |
3 | task:
4 | normalize: false
5 |
6 | model:
7 | w2v_path: /checkpoint/wnhsu/w2v/hubert_final/iter1/hubert.km.randcrop.pmw1_0.puw0_0.grpnorm.ml10.mp0_8.untie.mxsz250000.ufreq1.maxtok1400000.MU400k.s1337.ngpu32/checkpoint_last.pt
8 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/finetune/lm/ls_4gram.yaml:
--------------------------------------------------------------------------------
1 | # @package _global_
2 |
3 | criterion:
4 | wer_kenlm_model: /checkpoint/abdo/old_checkpoint02/datasets/librispeech/4-gram.bin
5 | wer_lexicon: /checkpoint/abdo/old_checkpoint02/datasets/librispeech/10h/raw/lexicon_ltr.lst
6 | wer_lm_weight: 2.0
7 | wer_word_score: -1.0
8 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/finetune/run/submitit_reg.yaml:
--------------------------------------------------------------------------------
1 | # @package _global_
2 |
3 | hydra:
4 | launcher:
5 | cpus_per_task: 8
6 | gpus_per_node: 8
7 | tasks_per_node: ${hydra.launcher.gpus_per_node}
8 | nodes: 1
9 | comment: null
10 | mem_gb: 384
11 | timeout_min: 4320
12 | max_num_timeout: 100
13 | constraint: volta32gb
14 | name: ${hydra.job.config_name}/${hydra.job.override_dirname}
15 | submitit_folder: ${hydra.sweep.dir}/submitit/%j
16 |
17 | distributed_training:
18 | distributed_world_size: 8
19 | distributed_port: 29671
20 | nprocs_per_node: 8
21 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/pretrain/data/iter1.yaml:
--------------------------------------------------------------------------------
1 | # @package _global_
2 |
3 | task:
4 | label_dir: ???
5 | labels: ["km"]
6 |
7 | model:
8 | label_rate: 100
9 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/pretrain/data/iter2.yaml:
--------------------------------------------------------------------------------
1 | # @package _global_
2 |
3 | task:
4 | label_dir: ???
5 | labels: ["km"]
6 |
7 | model:
8 | label_rate: 50
9 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/config/pretrain/run/submitit_reg.yaml:
--------------------------------------------------------------------------------
1 | # @package _global_
2 |
3 | hydra:
4 | launcher:
5 | cpus_per_task: 8
6 | gpus_per_node: 8
7 | tasks_per_node: ${hydra.launcher.gpus_per_node}
8 | nodes: 4
9 | comment: null
10 | mem_gb: 384
11 | timeout_min: 4320
12 | max_num_timeout: 100
13 | constraint: volta32gb
14 | name: ${hydra.job.config_name}/${hydra.job.override_dirname}
15 | submitit_folder: ${hydra.sweep.dir}/submitit/%j
16 |
17 | distributed_training:
18 | distributed_world_size: 32
19 | distributed_port: 29671
20 | nprocs_per_node: 8
21 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/hubert/update_ckpt.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import torch
7 |
8 | src_ckpt = "/checkpoint/wnhsu/w2v/archived/hubert_base_ls960_it2.pt"
9 | ref_ckpt = "/checkpoint/wnhsu/w2v/hubert_icassp_oss_v3/iter2_km100-400k-grp-L6/oss.km500_p0_1_s334.pmw1_0.puw0_0.grpnorm.ml10.mp0_8.untie.mxsz250000.ufreq1.maxtok1400000.MU100k.s1337.ngpu32/checkpoint_last.pt"
10 | new_ckpt = "/checkpoint/wnhsu/w2v/archived/hubert_base_ls960_it2_updated.pt"
11 |
12 |
13 | def update_state(state):
14 | state["model"]["label_embs_concat"] = state["model"].pop("label_embs")
15 | state["args"].task = "hubert_pretraining"
16 | state["args"].labels = f"['{state['args'].labels}']"
17 | return state
18 |
19 |
20 | src_state = torch.load(src_ckpt)
21 | src_state = update_state(src_state)
22 | torch.save(src_state, new_ckpt)
23 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/language_model/prepare-wikitext-103.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh
3 |
4 | URLS=(
5 | "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip"
6 | )
7 | FILES=(
8 | "wikitext-103-v1.zip"
9 | )
10 |
11 | for ((i=0;i<${#URLS[@]};++i)); do
12 | file=${FILES[i]}
13 | if [ -f $file ]; then
14 | echo "$file already exists, skipping download"
15 | else
16 | url=${URLS[i]}
17 | wget "$url"
18 | if [ -f $file ]; then
19 | echo "$url successfully downloaded."
20 | else
21 | echo "$url not successfully downloaded."
22 | exit -1
23 | fi
24 | if [ ${file: -4} == ".tgz" ]; then
25 | tar zxvf $file
26 | elif [ ${file: -4} == ".tar" ]; then
27 | tar xvf $file
28 | elif [ ${file: -4} == ".zip" ]; then
29 | unzip $file
30 | fi
31 | fi
32 | done
33 | cd ..
34 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/laser/laser_src/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .laser_task import * # noqa
7 | from .laser_lstm import * # noqa
8 | from .laser_transformer import * # noqa
9 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/latent_depth/latent_depth_src/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import multilingual_translation_latent_depth # noqa
7 | from .loss import latent_depth # noqa
8 | from .models import latent_multilingual_transformer # noqa
9 | from .modules import latent_layers # noqa
10 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/latent_depth/latent_depth_src/loss/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/latent_depth/latent_depth_src/loss/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/latent_depth/latent_depth_src/models/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/latent_depth/latent_depth_src/models/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/latent_depth/latent_depth_src/modules/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/latent_depth/latent_depth_src/modules/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/linformer/README.md:
--------------------------------------------------------------------------------
1 | # Linformer: Self-Attention with Linear Complexity (Wang et al., 2020)
2 |
3 | This example contains code to train Linformer models as described in our paper
4 | [Linformer: Self-Attention with Linear Complexity](https://arxiv.org/abs/2006.04768).
5 |
6 | ## Training a new Linformer RoBERTa model
7 |
8 | You can mostly follow the [RoBERTa pretraining README](/examples/roberta/README.pretraining.md),
9 | updating your training command with `--user-dir examples/linformer/linformer_src --arch linformer_roberta_base`.
10 |
11 | ## Citation
12 |
13 | If you use our work, please cite:
14 |
15 | ```bibtex
16 | @article{wang2020linformer,
17 | title={Linformer: Self-Attention with Linear Complexity},
18 | author={Wang, Sinong and Li, Belinda and Khabsa, Madian and Fang, Han and Ma, Hao},
19 | journal={arXiv preprint arXiv:2006.04768},
20 | year={2020}
21 | }
22 | ```
23 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/linformer/linformer_src/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .models import linformer_roberta # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/linformer/linformer_src/models/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/linformer/linformer_src/models/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/linformer/linformer_src/modules/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/linformer/linformer_src/modules/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/m2m_100/tokenizers/README.md:
--------------------------------------------------------------------------------
1 | # M2M-100 Tokenization
2 |
3 | We apply different tokenization strategies for different languages following the existing literature. Here we provide tok.sh a tokenizer that can be used to reproduce our results.
4 |
5 | To reproduce the results, follow these steps:
6 |
7 | ```
8 | tgt_lang=...
9 | reference_translation=...
10 | cat generation_output | grep -P "^H" | sort -V | cut -f 3- | sh tok.sh $tgt_lang > hyp
11 | cat $reference_translation |sh tok.sh $tgt_lang > ref
12 | sacrebleu -tok 'none' ref < hyp
13 | ```
14 |
15 | ## Installation
16 |
17 | Tools needed for all the languages except Arabic can be installed by running install_dependencies.sh
18 | If you want to evaluate Arabic models, please follow the instructions provided here: http://alt.qcri.org/tools/arabic-normalizer/ to install
19 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/m2m_100/tokenizers/seg_ja.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | SCRIPT=`realpath $0`
7 | KYTEA=`dirname $SCRIPT`/thirdparty/kytea
8 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$KYTEA/lib:/usr/local/lib
9 | export PATH=$PATH:"$KYTEA/bin"
10 |
11 | cat - | tr -d "[:blank:]" | kytea -notags
12 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/m2m_100/tokenizers/seg_ko.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | SCRIPT=`realpath $0`
7 | MECAB=`dirname $SCRIPT`/thirdparty/mecab-0.996-ko-0.9.2
8 |
9 | export PATH=$PATH:"$MECAB/bin":"$MECAB/lib"
10 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"$MECAB/lib"
11 |
12 | cat - | mecab -O wakati
13 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/m2m_100/tokenizers/thirdparty/.gitignore:
--------------------------------------------------------------------------------
1 | seg_my.py
2 | indic_nlp_library/
3 | indic_nlp_resources/
4 | kytea/
5 | mecab-0.996-ko-0.9.2.tar.gz
6 | mecab-0.996-ko-0.9.2/
7 | mosesdecoder/
8 | wat2020.my-en.zip
9 | wat2020.my-en/
10 | wmt16-scripts/
11 | mecab-ko-dic-2.1.1-20180720/
12 | mecab-ko-dic-2.1.1-20180720.tar.gz
--------------------------------------------------------------------------------
/Machine_Translation/examples/m2m_100/tokenizers/tokenize_indic.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | # Use: echo {text} | python tokenize_indic.py {language}
8 |
9 | import sys
10 |
11 | from indicnlp.normalize.indic_normalize import IndicNormalizerFactory
12 | from indicnlp.tokenize.indic_tokenize import trivial_tokenize
13 |
14 |
15 | factory = IndicNormalizerFactory()
16 | normalizer = factory.get_normalizer(
17 | sys.argv[1], remove_nuktas=False, nasals_mode="do_nothing"
18 | )
19 |
20 | for line in sys.stdin:
21 | normalized_line = normalizer.normalize(line.strip())
22 | tokenized_line = " ".join(trivial_tokenize(normalized_line, sys.argv[1]))
23 | print(tokenized_line)
24 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/m2m_100/tokenizers/tokenize_thai.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | import sys
8 |
9 | from pythainlp import word_tokenize
10 |
11 |
12 | for line in sys.stdin:
13 | print(" ".join(word_tokenize(line.strip())))
14 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/m2m_100/tokenizers/tokenize_zh.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 |
8 | import fileinput
9 |
10 | import sacrebleu
11 |
12 |
13 | for line in fileinput.input():
14 | print(sacrebleu.tokenize_zh(line))
15 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/m2m_100/tokenizers/tokenizer_ar.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env sh
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 | # Please follow the instructions here http://alt.qcri.org/tools/arabic-normalizer/
8 | # to install tools needed for Arabic
9 |
10 | echo "Please install Arabic tools: http://alt.qcri.org/tools/arabic-normalizer/"
11 | echo "Then update environment variables in tokenizer_ar.sh"
12 | exit 1
13 |
14 | SVMTOOL=...
15 | GOMOSESGO=...
16 | QCRI_ARABIC_NORMALIZER=...
17 |
18 | export PERL5LIB="$SVMTOOL/lib":"$GOMOSESGO/bin/MADA-3.2":$PERL5LIB
19 |
20 |
21 | tempfile=$(mktemp)
22 | cat - > $tempfile
23 |
24 | cd $QCRI_ARABIC_NORMALIZER
25 |
26 | bash qcri_normalizer_mada3.2_aramorph1.2.1.sh $tempfile
27 | cat $tempfile.mada_norm-aramorph.europarl_tok
28 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/megatron_11b/detok.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3 -u
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | import argparse
8 | import fileinput
9 |
10 | import sacremoses
11 |
12 |
13 | def main():
14 | parser = argparse.ArgumentParser(description="")
15 | parser.add_argument("files", nargs="*", help="input files")
16 | args = parser.parse_args()
17 |
18 | detok = sacremoses.MosesDetokenizer()
19 |
20 | for line in fileinput.input(args.files, openhook=fileinput.hook_compressed):
21 | print(
22 | detok.detokenize(line.strip().split(" "))
23 | .replace(" @", "")
24 | .replace("@ ", "")
25 | .replace(" =", "=")
26 | .replace("= ", "=")
27 | .replace(" – ", "–")
28 | )
29 |
30 |
31 | if __name__ == "__main__":
32 | main()
33 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/multilingual/ML50_langs.txt:
--------------------------------------------------------------------------------
1 | ar_AR
2 | cs_CZ
3 | de_DE
4 | en_XX
5 | es_XX
6 | et_EE
7 | fi_FI
8 | fr_XX
9 | gu_IN
10 | hi_IN
11 | it_IT
12 | ja_XX
13 | kk_KZ
14 | ko_KR
15 | lt_LT
16 | lv_LV
17 | my_MM
18 | ne_NP
19 | nl_XX
20 | ro_RO
21 | ru_RU
22 | si_LK
23 | tr_TR
24 | vi_VN
25 | zh_CN
26 | af_ZA
27 | az_AZ
28 | bn_IN
29 | fa_IR
30 | he_IL
31 | hr_HR
32 | id_ID
33 | ka_GE
34 | km_KH
35 | mk_MK
36 | ml_IN
37 | mn_MN
38 | mr_IN
39 | pl_PL
40 | ps_AF
41 | pt_XX
42 | sv_SE
43 | sw_KE
44 | ta_IN
45 | te_IN
46 | th_TH
47 | tl_XX
48 | uk_UA
49 | ur_PK
50 | xh_ZA
51 | gl_ES
52 | sl_SI
--------------------------------------------------------------------------------
/Machine_Translation/examples/multilingual/data_scripts/README.md:
--------------------------------------------------------------------------------
1 |
2 | # Install dependency
3 | ```bash
4 | pip install -r requirement.txt
5 | ```
6 |
7 | # Download the data set
8 | ```bash
9 | export WORKDIR_ROOT=
10 |
11 | ```
12 | The downloaded data will be at $WORKDIR_ROOT/ML50
13 |
14 | # preprocess the data
15 | Install SPM [here](https://github.com/google/sentencepiece)
16 | ```bash
17 | export WORKDIR_ROOT=
18 | export SPM_PATH=
19 | ```
20 | * $WORKDIR_ROOT/ML50/raw: extracted raw data
21 | * $WORKDIR_ROOT/ML50/dedup: dedup data
22 | * $WORKDIR_ROOT/ML50/clean: data with valid and test sentences removed from the dedup data
23 |
24 |
25 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/multilingual/data_scripts/download_ML50_v1.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | # All rights reserved.
4 | #
5 | # This source code is licensed under the license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 | if [ -z $WORKDIR_ROOT ] ;
9 | then
10 | echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..."
11 | exit
12 | fi
13 |
14 | # first run download_wmt20.sh; it will install a few useful tools for other scripts
15 | # TODO: need to print out instructions on downloading a few files which requires manually authentication from the websites
16 | bash ./download_wmt20.sh
17 |
18 | python ./download_wmt19_and_before.py
19 | bash ./download_wat19_my.sh
20 | python ./download_ted_and_extract.py
21 | bash ./download_lotus.sh
22 | bash ./download_iitb.sh
23 | bash ./download_af_xh.sh
24 |
25 |
26 | # IWSLT downloading URLs have changed in between; TODO: fix them:
27 | bash ./download_iwslt_and_extract.sh
28 |
29 | # TODO: globalvoices URLs changed; need to be fixed
30 | bash ./download_flores_data.sh
31 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/multilingual/data_scripts/download_iitb.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | # All rights reserved.
4 | #
5 | # This source code is licensed under the license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 |
9 | if [ -z $WORKDIR_ROOT ] ;
10 | then
11 | echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..."
12 | exit
13 | fi
14 |
15 | IITB=$WORKDIR_ROOT/IITB
16 | mkdir -p $IITB
17 | pushd $IITB
18 |
19 | wget http://www.cfilt.iitb.ac.in/~moses/iitb_en_hi_parallel/iitb_corpus_download/parallel.tgz
20 | tar -xvzf parallel.tgz
21 |
22 | wget http://www.cfilt.iitb.ac.in/~moses/iitb_en_hi_parallel/iitb_corpus_download/dev_test.tgz
23 | tar -xvzf dev_test.tgz
24 |
25 | DESTDIR=${WORKDIR_ROOT}/ML50/raw/
26 |
27 | cp parallel/IITB.en-hi.en $DESTDIR/train.hi_IN-en_XX.en_XX
28 | cp parallel/IITB.en-hi.hi $DESTDIR/train.hi_IN-en_XX.hi_IN
29 |
30 | cp dev_test/dev.en $DESTDIR/valid.hi_IN-en_XX.en_XX
31 | cp dev_test/dev.hi $DESTDIR/valid.hi_IN-en_XX.hi_IN
32 |
33 | cp dev_test/test.en $DESTDIR/test.hi_IN-en_XX.en_XX
34 | cp dev_test/test.hi $DESTDIR/test.hi_IN-en_XX.hi_IN
35 | popd
--------------------------------------------------------------------------------
/Machine_Translation/examples/multilingual/data_scripts/preprocess_ML50_v1.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | # All rights reserved.
4 | #
5 | # This source code is licensed under the license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 | if [ -z $WORKDIR_ROOT ] ;
9 | then
10 | echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..."
11 | exit
12 | fi
13 |
14 | if [ -z $SPM_PATH ] ;
15 | then
16 | echo "Please install sentence piecence from https://github.com/google/sentencepiece and set SPM_PATH pointing to the installed spm_encode.py. Exitting..."
17 | exit
18 | fi
19 |
20 | ML50=${WORKDIR_ROOT}/ML50
21 |
22 | mkdir -p $ML50/dedup
23 | mkdir -p $ML50/cleaned_dedup
24 |
25 | python ./dedup_all.py --from-folder $ML50/raw --to-folder $ML50/dedup
26 | python ./remove_valid_test_in_train.py --from-folder $ML50/dedup --to-folder $ML50/clean
27 | python ./binarize.py --raw-folder $ML50/clean
--------------------------------------------------------------------------------
/Machine_Translation/examples/multilingual/data_scripts/requirement.txt:
--------------------------------------------------------------------------------
1 | wget
2 | pandas
--------------------------------------------------------------------------------
/Machine_Translation/examples/multilingual/data_scripts/utils/strip_sgm.sh:
--------------------------------------------------------------------------------
1 | grep "seg id" | sed 's///g' | sed 's/<\/seg>//g'
2 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/multilingual/multilingual_fairseq_gen.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | # All rights reserved.
4 | #
5 | # This source code is licensed under the license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 | lang_pairs="en-fr,en-cs,fr-en,cs-en"
9 | path_2_data=$1 #
10 | lang_list=$2 #
11 | model=$3 #
12 | source_lang=cs
13 | target_lang=en
14 |
15 | fairseq-generate "$path_2_data" \
16 | --path "$model" \
17 | --task translation_multi_simple_epoch \
18 | --gen-subset test \
19 | --source-lang "$source_lang" \
20 | --target-lang "$target_lang" \
21 | --sacrebleu --remove-bpe 'sentencepiece'\
22 | --batch-size 32 \
23 | --encoder-langtok "src" \
24 | --decoder-langtok \
25 | --lang-dict "$lang_list" \
26 | --lang-pairs "$lang_pairs"
27 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/noisychannel/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .rerank_options import * # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/pointer_generator/pointer_generator_src/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import transformer_pg # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/roberta/commonsense_qa/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import commonsense_qa_task # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/roberta/commonsense_qa/download_cqa_data.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | OUTDIR=data/CommonsenseQA
8 |
9 | mkdir -p $OUTDIR
10 |
11 | wget -O $OUTDIR/train.jsonl https://s3.amazonaws.com/commensenseqa/train_rand_split.jsonl
12 | wget -O $OUTDIR/valid.jsonl https://s3.amazonaws.com/commensenseqa/dev_rand_split.jsonl
13 | wget -O $OUTDIR/test.jsonl https://s3.amazonaws.com/commensenseqa/test_rand_split_no_answers.jsonl
14 | wget -O $OUTDIR/dict.txt https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt
15 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/roberta/config/pretraining/base.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 | common:
3 | fp16: true
4 | log_format: json
5 | log_interval: 200
6 |
7 | checkpoint:
8 | no_epoch_checkpoints: true
9 |
10 | task:
11 | _name: masked_lm
12 | data: ???
13 | sample_break_mode: complete
14 | tokens_per_sample: 512
15 |
16 | criterion: masked_lm
17 |
18 | dataset:
19 | batch_size: 16
20 | ignore_unused_valid_subsets: true
21 |
22 | optimizer:
23 | _name: adam
24 | weight_decay: 0.01
25 | adam_betas: (0.9,0.98)
26 | adam_eps: 1e-06
27 |
28 | lr_scheduler:
29 | _name: polynomial_decay
30 | warmup_updates: 10000
31 |
32 | optimization:
33 | clip_norm: 0
34 | lr: [0.0005]
35 | max_update: 125000
36 | update_freq: [16]
37 |
38 | model:
39 | _name: roberta
40 | max_positions: 512
41 | dropout: 0.1
42 | attention_dropout: 0.1
43 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/roberta/wsc/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import wsc_criterion # noqa
7 | from . import wsc_task # noqa
8 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/rxf/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import rxf_src # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/rxf/rxf_src/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import label_smoothed_cross_entropy_r3f, sentence_prediction_r3f # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/simultaneous_translation/README.md:
--------------------------------------------------------------------------------
1 | # Simultaneous Translation
2 | Examples of simultaneous translation in fairseq
3 | - [English-to-Japanese text-to-text wait-k model](docs/enja-waitk.md)
4 | - [English-to-Germen text-to-text monotonic multihead attention model](docs/ende-mma.md)
5 | - [English-to-Germen speech-to-text simultaneous translation model](../speech_to_text/docs/simulst_mustc_example.md)
6 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/simultaneous_translation/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import models # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/simultaneous_translation/models/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import importlib
7 | import os
8 |
9 |
10 | for file in sorted(os.listdir(os.path.dirname(__file__))):
11 | if file.endswith(".py") and not file.startswith("_"):
12 | model_name = file[: file.find(".py")]
13 | importlib.import_module(
14 | "examples.simultaneous_translation.models." + model_name
15 | )
16 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/simultaneous_translation/modules/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 |
7 | import os
8 | import importlib
9 | from fairseq import registry
10 |
11 | (
12 | build_monotonic_attention,
13 | register_monotonic_attention,
14 | MONOTONIC_ATTENTION_REGISTRY,
15 | _,
16 | ) = registry.setup_registry("--simul-type")
17 |
18 | for file in sorted(os.listdir(os.path.dirname(__file__))):
19 | if file.endswith(".py") and not file.startswith("_"):
20 | model_name = file[: file.find(".py")]
21 | importlib.import_module(
22 | "examples.simultaneous_translation.modules." + model_name
23 | )
24 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/simultaneous_translation/utils/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import importlib
7 | import os
8 |
9 |
10 | # automatically import any Python files in the criterions/ directory
11 | for file in sorted(os.listdir(os.path.dirname(__file__))):
12 | if file.endswith(".py") and not file.startswith("_"):
13 | module = file[: file.find(".py")]
14 | importlib.import_module("examples.simultaneous_translation.utils." + module)
15 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/__init__.py:
--------------------------------------------------------------------------------
1 | from . import criterions, models, tasks # noqa
2 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/criterions/__init__.py:
--------------------------------------------------------------------------------
1 | import importlib
2 | import os
3 |
4 |
5 | # ASG loss requires flashlight bindings
6 | files_to_skip = set()
7 | try:
8 | import flashlight.lib.sequence.criterion
9 | except ImportError:
10 | files_to_skip.add("ASG_loss.py")
11 |
12 | for file in sorted(os.listdir(os.path.dirname(__file__))):
13 | if file.endswith(".py") and not file.startswith("_") and file not in files_to_skip:
14 | criterion_name = file[: file.find(".py")]
15 | importlib.import_module(
16 | "examples.speech_recognition.criterions." + criterion_name
17 | )
18 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/data/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .asr_dataset import AsrDataset
7 |
8 |
9 | __all__ = [
10 | "AsrDataset",
11 | ]
12 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/kaldi/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/speech_recognition/kaldi/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/kaldi/config/kaldi_initializer.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 |
3 | data_dir: ???
4 | fst_dir: ???
5 | in_labels: ???
6 | kaldi_root: ???
7 | lm_arpa: ???
8 | blank_symbol:
9 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/models/__init__.py:
--------------------------------------------------------------------------------
1 | import importlib
2 | import os
3 |
4 |
5 | for file in sorted(os.listdir(os.path.dirname(__file__))):
6 | if file.endswith(".py") and not file.startswith("_"):
7 | model_name = file[: file.find(".py")]
8 | importlib.import_module("examples.speech_recognition.models." + model_name)
9 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/new/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/speech_recognition/new/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/new/conf/hydra/sweeper/ax.yaml:
--------------------------------------------------------------------------------
1 | # @package hydra.sweeper
2 | _target_: hydra_plugins.hydra_ax_sweeper.ax_sweeper.AxSweeper
3 | max_batch_size: null
4 | ax_config:
5 | max_trials: 128
6 | early_stop:
7 | minimize: true
8 | max_epochs_without_improvement: 32
9 | epsilon: 1.0e-05
10 | experiment:
11 | name: ${dataset.gen_subset}
12 | objective_name: wer
13 | minimize: true
14 | parameter_constraints: null
15 | outcome_constraints: null
16 | status_quo: null
17 | client:
18 | verbose_logging: false
19 | random_seed: null
20 | params:
21 | decoding.lmweight:
22 | type: range
23 | bounds: [0.0, 5.0]
24 | decoding.wordscore:
25 | type: range
26 | bounds: [-5.0, 5.0]
27 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/new/conf/infer.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 |
3 | defaults:
4 | - task: null
5 | - model: null
6 |
7 | hydra:
8 | run:
9 | dir: ${common_eval.results_path}/${dataset.gen_subset}
10 | sweep:
11 | dir: ${common_eval.results_path}
12 | subdir: ${dataset.gen_subset}
13 | common_eval:
14 | results_path: null
15 | path: null
16 | post_process: letter
17 | quiet: true
18 | dataset:
19 | max_tokens: 1000000
20 | gen_subset: test
21 | distributed_training:
22 | distributed_world_size: 1
23 | decoding:
24 | beam: 5
25 | type: viterbi
26 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/new/decoders/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/speech_recognition/new/decoders/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/new/decoders/decoder.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | # Copyright (c) Facebook, Inc. and its affiliates.
4 | #
5 | # This source code is licensed under the MIT license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 | from typing import Union
9 |
10 | from fairseq.data.dictionary import Dictionary
11 |
12 | from .decoder_config import DecoderConfig, FlashlightDecoderConfig
13 | from .base_decoder import BaseDecoder
14 |
15 |
16 | def Decoder(
17 | cfg: Union[DecoderConfig, FlashlightDecoderConfig], tgt_dict: Dictionary
18 | ) -> BaseDecoder:
19 |
20 | if cfg.type == "viterbi":
21 | from .viterbi_decoder import ViterbiDecoder
22 |
23 | return ViterbiDecoder(tgt_dict)
24 | if cfg.type == "kenlm":
25 | from .flashlight_decoder import KenLMDecoder
26 |
27 | return KenLMDecoder(cfg, tgt_dict)
28 | if cfg.type == "fairseqlm":
29 | from .flashlight_decoder import FairseqLMDecoder
30 |
31 | return FairseqLMDecoder(cfg, tgt_dict)
32 | raise NotImplementedError(f"Invalid decoder name: {cfg.name}")
33 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/new/decoders/viterbi_decoder.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | # Copyright (c) Facebook, Inc. and its affiliates.
4 | #
5 | # This source code is licensed under the MIT license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 | import torch
9 |
10 | from typing import List, Dict
11 |
12 | from .base_decoder import BaseDecoder
13 |
14 |
15 | class ViterbiDecoder(BaseDecoder):
16 | def decode(
17 | self,
18 | emissions: torch.FloatTensor,
19 | ) -> List[List[Dict[str, torch.LongTensor]]]:
20 | def get_pred(e):
21 | toks = e.argmax(dim=-1).unique_consecutive()
22 | return toks[toks != self.blank]
23 |
24 | return [[{"tokens": get_pred(x), "score": 0}] for x in emissions]
25 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_recognition/tasks/__init__.py:
--------------------------------------------------------------------------------
1 | import importlib
2 | import os
3 |
4 |
5 | for file in sorted(os.listdir(os.path.dirname(__file__))):
6 | if file.endswith(".py") and not file.startswith("_"):
7 | task_name = file[: file.find(".py")]
8 | importlib.import_module("examples.speech_recognition.tasks." + task_name)
9 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_text_joint_to_text/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import tasks, criterions, models # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_text_joint_to_text/configs/mustc_noise.list:
--------------------------------------------------------------------------------
1 | "(Applause) NOISE
2 | "(Laughter) VOICE
3 | "(Laughter)" VOICE
4 | (Applause) NOISE
5 | (Applause). NOISE
6 | (Audience) VOICE
7 | (Audio) NOISE
8 | (Beat) NOISE
9 | (Beatboxing) VOICE
10 | (Beep) NOISE
11 | (Beeps) NOISE
12 | (Cheering) VOICE
13 | (Cheers) VOICE
14 | (Claps) NOISE
15 | (Clicking) NOISE
16 | (Clunk) NOISE
17 | (Coughs) NOISE
18 | (Drums) NOISE
19 | (Explosion) NOISE
20 | (Gasps) VOICE
21 | (Guitar) NOISE
22 | (Honk) NOISE
23 | (Laugher) VOICE
24 | (Laughing) VOICE
25 | (Laughs) VOICE
26 | (Laughter) VOICE
27 | (Laughter). VOICE
28 | (Laughter)... VOICE
29 | (Mumbling) VOICE
30 | (Music) NOISE
31 | (Noise) NOISE
32 | (Recording) VOICE
33 | (Ringing) NOISE
34 | (Shouts) VOICE
35 | (Sigh) VOICE
36 | (Sighs) VOICE
37 | (Silence) NOISE
38 | (Singing) VOICE
39 | (Sings) VOICE
40 | (Spanish) VOICE
41 | (Static) NOISE
42 | (Tones) NOISE
43 | (Trumpet) NOISE
44 | (Video) NOISE
45 | (Video): NOISE
46 | (Voice-over) NOISE
47 | (Whistle) NOISE
48 | (Whistling) NOISE
49 | (video): NOISE
50 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_text_joint_to_text/criterions/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import importlib
7 | import os
8 |
9 |
10 | for file in os.listdir(os.path.dirname(__file__)):
11 | if file.endswith(".py") and not file.startswith("_"):
12 | criterion_name = file[: file.find(".py")]
13 | importlib.import_module(
14 | "examples.speech_text_joint_to_text.criterions." + criterion_name
15 | )
16 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_text_joint_to_text/models/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import importlib
7 | import os
8 |
9 | for file in os.listdir(os.path.dirname(__file__)):
10 | if file.endswith(".py") and not file.startswith("_"):
11 | model_name = file[: file.find(".py")]
12 | importlib.import_module(
13 | "examples.speech_text_joint_to_text.models." + model_name
14 | )
15 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/speech_text_joint_to_text/tasks/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import importlib
7 | import os
8 |
9 | for file in os.listdir(os.path.dirname(__file__)):
10 | if file.endswith(".py") and not file.startswith("_"):
11 | task_name = file[: file.find(".py")]
12 | importlib.import_module("examples.speech_text_joint_to_text.tasks." + task_name)
13 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/translation_moe/translation_moe_src/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import translation_moe # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/translation_moe/translation_moe_src/logsumexp_moe.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import torch
7 |
8 |
9 | class LogSumExpMoE(torch.autograd.Function):
10 | """Standard LogSumExp forward pass, but use *posterior* for the backward.
11 |
12 | See `"Mixture Models for Diverse Machine Translation: Tricks of the Trade"
13 | (Shen et al., 2019) `_.
14 | """
15 |
16 | @staticmethod
17 | def forward(ctx, logp, posterior, dim=-1):
18 | ctx.save_for_backward(posterior)
19 | ctx.dim = dim
20 | return torch.logsumexp(logp, dim=dim)
21 |
22 | @staticmethod
23 | def backward(ctx, grad_output):
24 | (posterior,) = ctx.saved_tensors
25 | grad_logp = grad_output.unsqueeze(ctx.dim) * posterior
26 | return grad_logp, None, None
27 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/truncated_bptt/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import transformer_xl_model, truncated_bptt_lm_task # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/unsupervised_quality_estimation/repeat_lines.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import argparse
7 | import sys
8 |
9 |
10 | def _normalize_spaces(line):
11 | return " ".join(line.split())
12 |
13 |
14 | def main():
15 | parser = argparse.ArgumentParser()
16 | parser.add_argument("-i", "--input_file", required=True, type=str)
17 | parser.add_argument("-n", "--repeat_times", required=True, type=int)
18 | parser.add_argument("-o", "--output_file", required=False, type=str)
19 | args = parser.parse_args()
20 | stream = open(args.output_file, "w") if args.output_file else sys.stdout
21 |
22 | for line in open(args.input_file):
23 | for _ in range(args.repeat_times):
24 | stream.write(_normalize_spaces(line) + "\n")
25 |
26 |
27 | if __name__ == "__main__":
28 | main()
29 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/wav2vec/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/examples/wav2vec/unsupervised/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/config/generate/viterbi.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 |
3 | fairseq:
4 | task:
5 | _name: unpaired_audio_text
6 | labels: phn
7 | data: ???
8 | sort_by_length: false
9 | shuffle: false
10 | text_data: ''
11 |
12 | common_eval:
13 | path: ???
14 | quiet: true
15 |
16 | dataset:
17 | gen_subset: valid
18 | batch_size: 1
19 |
20 | w2l_decoder: VITERBI
21 | post_process: silence
22 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/data/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .extracted_features_dataset import ExtractedFeaturesDataset
7 | from .random_input_dataset import RandomInputDataset
8 |
9 |
10 | __all__ = [
11 | "ExtractedFeaturesDataset",
12 | "RandomInputDataset",
13 | ]
14 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/cmd.sh:
--------------------------------------------------------------------------------
1 | # you can change cmd.sh depending on what type of queue you are using.
2 | # If you have no queueing system and want to run on a local machine, you
3 | # can change all instances 'queue.pl' to run.pl (but be careful and run
4 | # commands one by one: most recipes will exhaust the memory on your
5 | # machine). queue.pl works with GridEngine (qsub). slurm.pl works
6 | # with slurm. Different queues are configured differently, with different
7 | # queue names and different ways of specifying things like memory;
8 | # to account for these differences you can create and edit the file
9 | # conf/queue.conf to match your queue's configuration. Search for
10 | # conf/queue.conf in http://kaldi-asr.org/doc/queue.html for more information,
11 | # or search for the string 'default_config' in utils/queue.pl or utils/slurm.pl.
12 |
13 | export train_cmd="run.pl --mem 2G"
14 | export decode_cmd="run.pl --mem 4G"
15 | export mkgraph_cmd="run.pl --mem 8G"
16 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_phone.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # decode into phones (and prepare a new data directory for HMM outputs)
4 |
5 | . ./path.sh
6 |
7 | set -eu
8 |
9 | out_dir= # same as in train.sh
10 | dec_lmparam= # LM hyperparameters (e.g., 7.0.0)
11 | dec_exp=
12 | dec_script=
13 | dec_splits="train valid"
14 | dec_data_dir=$out_dir/dec_data # where to write HMM output
15 |
16 | data_dir=${out_dir}/data
17 |
18 | local/decode.sh --nj 40 --graph_name graph \
19 | --val_sets "$dec_splits" --decode_script $dec_script \
20 | $out_dir/exp/$dec_exp $data_dir $data_dir/lang_test
21 |
22 | if [ ! -z $dec_lmparam ]; then
23 | for x in $dec_splits; do
24 | mkdir -p $dec_data_dir/$x
25 | cp $data_dir/$x/{feats.scp,cmvn.scp,utt2spk,spk2utt} $dec_data_dir/$x/
26 |
27 | tra=$out_dir/exp/$dec_exp/decode_${x}/scoring/${dec_lmparam}.tra
28 | cat $tra | utils/int2sym.pl -f 2- $data_dir/lang/words.txt | \
29 | sed 's:::g' | sed 's:::g' > $dec_data_dir/${x}/text
30 | utils/fix_data_dir.sh $dec_data_dir/${x}
31 | echo "WER on ${x} is" $(compute-wer ark:$data_dir/${x}_gt/text ark:$dec_data_dir/$x/text | cut -d" " -f2-)
32 | done
33 | fi
34 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_word_step2.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # prepare a new data directory of HMM word output
4 |
5 | . ./path.sh
6 |
7 | set -eu
8 |
9 | out_dir= # same as in train.sh
10 | dec_lmparam= # LM hyperparameters (e.g., 7.0.0)
11 |
12 | dec_exp=tri3b # what HMM stage to decode (e.g., tri3b)
13 | dec_suffix=word
14 | dec_splits="train valid"
15 | dec_data_dir=$out_dir/dec_data_word # where to write HMM output
16 |
17 | data_dir=$out_dir/data
18 | wrd_data_dir=$out_dir/data_word
19 |
20 | for x in $dec_splits; do
21 | mkdir -p $dec_data_dir/$x
22 | cp $data_dir/$x/{feats.scp,cmvn.scp,utt2spk,spk2utt} $dec_data_dir/$x/
23 |
24 | tra=$out_dir/exp/$dec_exp/decode${dec_suffix}_${x}/scoring/${dec_lmparam}.tra
25 | cat $tra | utils/int2sym.pl -f 2- $data_dir/lang_word/words.txt | \
26 | sed 's:::g' | sed 's:::g' > $dec_data_dir/$x/text
27 | utils/fix_data_dir.sh $dec_data_dir/$x
28 | echo "WER on $x is" $(compute-wer ark:$wrd_data_dir/${x}_gt/text ark:$dec_data_dir/$x/text | cut -d" " -f2-)
29 | done
30 |
31 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/copy_aligned_text.py:
--------------------------------------------------------------------------------
1 | import sys
2 |
3 | for idx, line in enumerate(sys.stdin):
4 | print(f"utt{idx:010d} {line}", end='')
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/decode.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -u
4 |
5 | val_sets="dev_other"
6 | graph_name=graph
7 | decode_suffix=""
8 | decode_script="steps/decode_fmllr.sh"
9 | decode_args=""
10 | nj=60
11 |
12 | . ./cmd.sh
13 | . ./path.sh
14 | . parse_options.sh
15 |
16 | set -x
17 | exp_dir=$1
18 | data_root=$2
19 | lang_test=$3
20 |
21 | graph=$exp_dir/$graph_name
22 |
23 | if [ ! -d $graph ]; then
24 | utils/mkgraph.sh $lang_test $exp_dir $graph
25 | fi
26 |
27 | for part in $val_sets; do
28 | dec_dir=$exp_dir/decode${decode_suffix}_${part}
29 | if [ ! -d $dec_dir ]; then
30 | echo "decoding $part for $exp_dir"
31 | $decode_script --nj $nj --cmd "$decode_cmd" $decode_args \
32 | $graph $data_root/$part $dec_dir &
33 | else
34 | echo "$dec_dir exists. skip"
35 | fi
36 | done
37 |
38 | wait
39 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lang.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | sil_prob=0.5
4 | num_sil_states=3
5 | num_nonsil_states=1
6 |
7 | . ./cmd.sh
8 | . ./path.sh
9 | . parse_options.sh
10 |
11 | set -eux
12 |
13 | dict=$1
14 | data_dir=$2
15 |
16 | dict_dir=$data_dir/local/dict
17 | tmplm_dir=$data_dir/local/lang_tmp
18 | lm_dir=$data_dir/lang
19 |
20 | mkdir -p $dict_dir $tmplm_dir $lm_dir
21 |
22 | # prepare dict
23 | echo "SIL" > $dict_dir/silence_phones.txt
24 | echo "SIL" > $dict_dir/optional_silence.txt
25 | awk '{print $1}' $dict > $dict_dir/nonsilence_phones.txt
26 |
27 | echo "SIL SIL" > $dict_dir/lexicon.txt
28 | echo " SIL" >> $dict_dir/lexicon.txt
29 | awk '{print $1" "$1}' $dict >> $dict_dir/lexicon.txt
30 |
31 | echo "SIL" > $dict_dir/extra_questions.txt
32 | awk '{printf $1" "} END {printf "\n"}' $dict >> $dict_dir/extra_questions.txt
33 |
34 | # prepare lang
35 | utils/prepare_lang.sh --sil-prob $sil_prob --position-dependent-phones false \
36 | --num_sil_states $num_sil_states --num_nonsil_states $num_nonsil_states \
37 | $dict_dir "" $tmplm_dir $lm_dir
38 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lang_word.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | num_sil_states=3
4 | num_nonsil_states=1
5 |
6 | . ./cmd.sh
7 | . ./path.sh
8 | . parse_options.sh
9 |
10 | set -eux
11 |
12 | dict=$1
13 | data_dir=$2
14 | lexicon=$3
15 |
16 | dict_dir=$data_dir/local/dict_word
17 | tmplm_dir=$data_dir/local/lang_tmp_word
18 | lm_dir=$data_dir/lang_word
19 |
20 | mkdir -p $dict_dir $tmplm_dir $lm_dir
21 |
22 | # prepare dict
23 | echo "SIL" > $dict_dir/silence_phones.txt
24 | echo "SIL" > $dict_dir/optional_silence.txt
25 | awk '{print $1}' $dict > $dict_dir/nonsilence_phones.txt
26 |
27 | (echo "!SIL SIL"; echo " SIL";) | cat - $lexicon > $dict_dir/lexicon.txt
28 |
29 | echo "SIL" > $dict_dir/extra_questions.txt
30 | awk '{printf $1" "} END {printf "\n"}' $dict >> $dict_dir/extra_questions.txt
31 |
32 | # prepare lang
33 | utils/prepare_lang.sh --position-dependent-phones false \
34 | --num_sil_states $num_sil_states --num_nonsil_states $num_nonsil_states \
35 | $dict_dir "" $tmplm_dir $lm_dir
36 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lm.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | langdir=""
4 | lmdir=""
5 |
6 | . ./cmd.sh
7 | . ./path.sh
8 | . parse_options.sh
9 |
10 | arpa_lm=$1
11 | data=$2
12 |
13 | if [ -z $langdir ]; then
14 | langdir=$data/lang
15 | fi
16 | if [ -z $lmdir ]; then
17 | lmdir=$data/lang_test
18 | fi
19 |
20 | if [ ! -d $langdir ]; then
21 | echo "$langdir not found. run local/prepare_lang.sh first" && exit 1
22 | fi
23 |
24 | mkdir -p $lmdir
25 | cp -r $langdir/* $lmdir
26 |
27 | if [[ "$arpa_lm" == *.gz ]]; then
28 | gunzip -c $arpa_lm | arpa2fst --disambig-symbol=#0 --read-symbol-table=$lmdir/words.txt - $lmdir/G.fst
29 | else
30 | arpa2fst --disambig-symbol=#0 --read-symbol-table=$lmdir/words.txt $arpa_lm $lmdir/G.fst
31 | fi
32 | fstisstochastic $lmdir/G.fst
33 | utils/validate_lang.pl $lmdir || exit 1
34 |
35 | echo "done preparing lm ($lmdir)"
36 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select_decode.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | split="dev_other"
4 | ref_txt="" # ground truth transcript path
5 | psd_txt="" # pseudo transcript path
6 | get_best_wer=true
7 | dec_name="decode"
8 | graph_name="graph"
9 | kenlm_path=/checkpoint/abaevski/data/speech/libri/librispeech_lm_novox.phnc_o6.bin
10 |
11 | . ./cmd.sh
12 | . ./path.sh
13 | . parse_options.sh
14 |
15 | exp_root=$1
16 | unsup_args=""
17 | if [ $# -ge 2 ]; then
18 | unsup_args=$2
19 | fi
20 |
21 | set -eu
22 |
23 | if [ ! -z $ref_txt ] && $get_best_wer; then
24 | echo "==== WER w.r.t. real transcript (select based on unsupervised metric)"
25 | for x in $exp_root/*/${dec_name}_${split}*; do
26 | lang=$(dirname $x)/$graph_name
27 |
28 | (
29 | for tra in $x/scoring/*.tra; do
30 | cat $tra | utils/int2sym.pl -f 2- $lang/words.txt | sed 's:::g' | sed 's:::g' > $tra.txt
31 | python local/unsup_select.py $psd_txt $tra.txt --kenlm_path $kenlm_path --gt_tra $ref_txt $unsup_args
32 | done 2>/dev/null | grep "score=" | sed 's/=/ /g' | sed 's/;//g' | sort -k3n | head -n1
33 | ) &
34 | done
35 | fi
36 | wait
37 |
38 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select_decode_word.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | split="dev_other"
4 | ref_txt="" # ground truth transcript path
5 | psd_txt="" # pseudo transcript path
6 | get_best_wer=true
7 | dec_name="decode"
8 | graph_name="graph"
9 | kenlm_path=/checkpoint/abaevski/data/speech/libri/librispeech_lm_novox.phnc_o6.bin
10 | phonemize_lexicon=""
11 |
12 | . ./cmd.sh
13 | . ./path.sh
14 | . parse_options.sh
15 | . /private/home/wnhsu/unsup_asr/fairseq-py-unsup/env.sh
16 |
17 | exp_root=$1
18 |
19 | set -eu
20 |
21 | if [ ! -z $ref_txt ] && $get_best_wer; then
22 | echo "==== WER w.r.t. real transcript (select based on unsupervised metric)"
23 | for x in $exp_root/*/${dec_name}_${split}*; do
24 | lang=$(dirname $x)/$graph_name
25 |
26 | for tra in $x/scoring/*.tra; do
27 | cat $tra | utils/int2sym.pl -f 2- $lang/words.txt | sed 's:\::g' > $tra.txt
28 | python local/unsup_select.py $psd_txt $tra.txt \
29 | --kenlm_path $kenlm_path --gt_tra $ref_txt --phonemize \
30 | --phonemize_lexicon "$phonemize_lexicon"
31 | done | grep "score=" | sed 's/=/ /g' | sed 's/;//g' | sort -k3n | head -n1
32 | done
33 | fi
34 |
35 |
36 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/path.sh:
--------------------------------------------------------------------------------
1 | export KALDI_ROOT=`pwd`/../../..
2 | export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PWD:$PATH
3 | [ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1
4 | . $KALDI_ROOT/tools/config/common_path.sh
5 | export LC_ALL=C
6 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/steps:
--------------------------------------------------------------------------------
1 | ../../wsj/s5/steps
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/kaldi_self_train/st/utils:
--------------------------------------------------------------------------------
1 | ../../wsj/s5/utils
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/models/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .wav2vec_u import Wav2vec_U
7 |
8 |
9 | __all__ = [
10 | "Wav2vec_U",
11 | ]
12 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/scripts/copy_labels.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3 -u
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | import sys
8 |
9 | for idx, line in enumerate(sys.stdin):
10 | print(f"utt{idx:010d} {line}", end="")
11 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/scripts/filter_lexicon.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3 -u
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | import argparse
8 | import sys
9 |
10 | from fairseq.data import Dictionary
11 |
12 |
13 | def get_parser():
14 | parser = argparse.ArgumentParser(
15 | description="filters a lexicon given a unit dictionary"
16 | )
17 | parser.add_argument("-d", "--unit-dict", help="unit dictionary", required=True)
18 | return parser
19 |
20 |
21 | def main():
22 | parser = get_parser()
23 | args = parser.parse_args()
24 |
25 | d = Dictionary.load(args.unit_dict)
26 | symbols = set(d.symbols)
27 |
28 | for line in sys.stdin:
29 | items = line.rstrip().split()
30 | skip = len(items) < 2
31 | for x in items[1:]:
32 | if x not in symbols:
33 | skip = True
34 | break
35 | if not skip:
36 | print(line, end="")
37 |
38 |
39 | if __name__ == "__main__":
40 | main()
41 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/scripts/filter_tsv.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3 -u
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | import os
8 | import argparse
9 | import sys
10 |
11 |
12 | parser = argparse.ArgumentParser()
13 | parser.add_argument("--tsv", required=True, type=str)
14 | parser.add_argument("--no-skip", action="store_true")
15 | parser.add_argument("--keep", action="store_true")
16 | params = parser.parse_args()
17 |
18 |
19 | def get_fname(line):
20 | p = os.path.basename(line.split("\t")[0])
21 | p = os.path.splitext(p)[0]
22 | return p
23 |
24 |
25 | # filenames to exclude
26 | seen = set()
27 | with open(params.tsv) as f:
28 | if not params.no_skip:
29 | root = next(f).rstrip()
30 | for line in f:
31 | seen.add(get_fname(line))
32 |
33 | for i, line in enumerate(sys.stdin):
34 | exists = get_fname(line) in seen
35 | keep = (exists and params.keep) or (not exists and not params.keep)
36 | if i == 0 or keep:
37 | print(line, end="")
38 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/scripts/ltr_to_wrd.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3 -u
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | import sys
8 |
9 |
10 | def main():
11 | for line in sys.stdin:
12 | print(line.replace(" ", "").replace("|", " ").strip())
13 |
14 |
15 | if __name__ == "__main__":
16 | main()
17 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/scripts/normalize_text.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | import regex
8 | import sys
9 |
10 |
11 | def main():
12 | filter_r = regex.compile(r"[^\p{L}\p{N}\p{M}\' \-]")
13 |
14 | for line in sys.stdin:
15 | line = line.strip()
16 | line = filter_r.sub(" ", line)
17 | line = " ".join(line.split())
18 | print(line)
19 |
20 |
21 | if __name__ == "__main__":
22 | main()
23 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/scripts/wrd_to_ltr.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3 -u
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | import sys
8 |
9 |
10 | def main():
11 | for line in sys.stdin:
12 | print(" ".join(list(line.strip().replace(" ", "|"))) + " |")
13 |
14 |
15 | if __name__ == "__main__":
16 | main()
17 |
--------------------------------------------------------------------------------
/Machine_Translation/examples/wav2vec/unsupervised/tasks/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .unpaired_audio_text import UnpairedAudioText
7 |
8 |
9 | __all__ = [
10 | "UnpairedAudioText",
11 | ]
12 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/benchmark/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | # import models/tasks to register them
7 | from . import dummy_dataset, dummy_lm, dummy_masked_lm, dummy_model, dummy_mt # noqa
8 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/benchmark/dummy_dataset.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from fairseq.data import FairseqDataset
3 |
4 |
5 | class DummyDataset(FairseqDataset):
6 | def __init__(self, batch, num_items, item_size):
7 | super().__init__()
8 | self.batch = batch
9 | self.num_items = num_items
10 | self.item_size = item_size
11 |
12 | def __getitem__(self, index):
13 | return index
14 |
15 | def __len__(self):
16 | return self.num_items
17 |
18 | def collater(self, samples):
19 | return self.batch
20 |
21 | @property
22 | def sizes(self):
23 | return np.array([self.item_size] * self.num_items)
24 |
25 | def num_tokens(self, index):
26 | return self.item_size
27 |
28 | def size(self, index):
29 | return self.item_size
30 |
31 | def ordered_indices(self):
32 | return np.arange(self.num_items)
33 |
34 | @property
35 | def supports_prefetch(self):
36 | return False
37 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/clib/libbleu/module.cpp:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2017-present, Facebook, Inc.
3 | * All rights reserved.
4 | *
5 | * This source code is licensed under the license found in the
6 | * LICENSE file in the root directory of this source tree.
7 | */
8 |
9 | #include
10 |
11 | static PyMethodDef method_def[] = {{NULL, NULL, 0, NULL}}; // NOLINT
12 |
13 | static struct PyModuleDef module_def = {
14 | PyModuleDef_HEAD_INIT,
15 | "libbleu", /* name of module */
16 | // NOLINTNEXTLINE
17 | NULL, /* module documentation, may be NULL */
18 | -1, /* size of per-interpreter state of the module,
19 | or -1 if the module keeps state in global variables. */
20 | method_def}; // NOLINT
21 |
22 | #if PY_MAJOR_VERSION == 2
23 | PyMODINIT_FUNC init_libbleu()
24 | #else
25 | PyMODINIT_FUNC PyInit_libbleu()
26 | #endif
27 | {
28 | PyObject* m = PyModule_Create(&module_def);
29 | if (!m) {
30 | return NULL;
31 | }
32 | return m;
33 | }
34 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/clib/libnat_cuda/edit_dist.h:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2017-present, Facebook, Inc.
3 | * All rights reserved.
4 | *
5 | * This source code is licensed under the license found in the
6 | * LICENSE file in the root directory of this source tree.
7 | */
8 |
9 | #pragma once
10 |
11 | #include
12 |
13 | torch::Tensor LevenshteinDistanceCuda(
14 | torch::Tensor source,
15 | torch::Tensor target,
16 | torch::Tensor source_length,
17 | torch::Tensor target_length);
18 |
19 | torch::Tensor GenerateDeletionLabelCuda(
20 | torch::Tensor source,
21 | torch::Tensor operations);
22 |
23 | std::pair GenerateInsertionLabelCuda(
24 | torch::Tensor source,
25 | torch::Tensor operations);
26 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/config.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 |
3 | hydra:
4 | run:
5 | dir: .
6 |
7 | defaults:
8 | - _self_
9 | - task: null
10 | - model: null
11 | - criterion: cross_entropy
12 | - optimizer: null
13 | - lr_scheduler: fixed
14 | - bpe: null
15 | - tokenizer: null
16 | - scoring: null
17 | - generation: null
18 | - common_eval: null
19 | - eval_lm: null
20 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_baevski_gbw.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 | activation_fn: "relu"
3 | dropout: 0.1
4 | attention_dropout: 0.1
5 | activation_dropout: 0.0
6 | relu_dropout: 0.0
7 | decoder_embed_dim: 512
8 | decoder_output_dim: 512
9 | decoder_input_dim: 512
10 | decoder_ffn_embed_dim: 4096
11 | decoder_layers: 12
12 | decoder_attention_heads: 16
13 | decoder_normalize_before: true
14 | no_decoder_final_norm: true
15 | adaptive_softmax_cutoff: null
16 | adaptive_softmax_dropout: 0
17 | adaptive_softmax_factor: 4
18 | no_token_positional_embeddings: false
19 | share_decoder_input_output_embed: false
20 | character_embeddings: false
21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]"
22 | character_embedding_dim: 4
23 | char_embedder_highway_layers: 2
24 | adaptive_input: false
25 | adaptive_input_factor: 4
26 | adaptive_input_cutoff: null
27 | tie_adaptive_weights: false
28 | tie_adaptive_proj: false
29 | decoder_learned_pos: false
30 | decoder_layerdrop: 0
31 | decoder_layers_to_keep: null
32 | layernorm_embedding: false
33 | no_scale_embedding: false
34 | quant_noise_pq: 0
35 | quant_noise_pq_block_size: 8
36 | quant_noise_scalar: 0
37 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_big.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 | activation_fn: "relu"
3 | dropout: 0.1
4 | attention_dropout: 0.0
5 | activation_dropout: 0.0
6 | relu_dropout: 0.0
7 | decoder_embed_dim: 1024
8 | decoder_output_dim: 1024
9 | decoder_input_dim: 1024
10 | decoder_ffn_embed_dim: 4096
11 | decoder_layers: 12
12 | decoder_attention_heads: 16
13 | decoder_normalize_before: true
14 | no_decoder_final_norm: false
15 | adaptive_softmax_cutoff: null
16 | adaptive_softmax_dropout: 0
17 | adaptive_softmax_factor: 4
18 | no_token_positional_embeddings: false
19 | share_decoder_input_output_embed: false
20 | character_embeddings: false
21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]"
22 | character_embedding_dim: 4
23 | char_embedder_highway_layers: 2
24 | adaptive_input: false
25 | adaptive_input_factor: 4
26 | adaptive_input_cutoff: null
27 | tie_adaptive_weights: false
28 | tie_adaptive_proj: false
29 | decoder_learned_pos: false
30 | decoder_layerdrop: 0
31 | decoder_layers_to_keep: null
32 | layernorm_embedding: false
33 | no_scale_embedding: false
34 | quant_noise_pq: 0
35 | quant_noise_pq_block_size: 8
36 | quant_noise_scalar: 0
37 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_gbw.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 | activation_fn: "relu"
3 | dropout: 0.1
4 | attention_dropout: 0.1
5 | activation_dropout: 0.0
6 | relu_dropout: 0.0
7 | decoder_embed_dim: 512
8 | decoder_output_dim: 512
9 | decoder_input_dim: 512
10 | decoder_ffn_embed_dim: 4096
11 | decoder_layers: 12
12 | decoder_attention_heads: 16
13 | decoder_normalize_before: true
14 | no_decoder_final_norm: true
15 | adaptive_softmax_cutoff: null
16 | adaptive_softmax_dropout: 0
17 | adaptive_softmax_factor: 4
18 | no_token_positional_embeddings: false
19 | share_decoder_input_output_embed: false
20 | character_embeddings: false
21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]"
22 | character_embedding_dim: 4
23 | char_embedder_highway_layers: 2
24 | adaptive_input: false
25 | adaptive_input_factor: 4
26 | adaptive_input_cutoff: null
27 | tie_adaptive_weights: false
28 | tie_adaptive_proj: false
29 | decoder_learned_pos: false
30 | decoder_layerdrop: 0
31 | decoder_layers_to_keep: null
32 | layernorm_embedding: false
33 | no_scale_embedding: false
34 | quant_noise_pq: 0
35 | quant_noise_pq_block_size: 8
36 | quant_noise_scalar: 0
37 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_gpt.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 | activation_fn: "gelu"
3 | dropout: 0.1
4 | attention_dropout: 0.1
5 | activation_dropout: 0.0
6 | relu_dropout: 0.0
7 | decoder_embed_dim: 768
8 | decoder_output_dim: 768
9 | decoder_input_dim: 768
10 | decoder_ffn_embed_dim: 3072
11 | decoder_layers: 12
12 | decoder_attention_heads: 12
13 | decoder_normalize_before: true
14 | no_decoder_final_norm: false
15 | adaptive_softmax_cutoff: null
16 | adaptive_softmax_dropout: 0
17 | adaptive_softmax_factor: 4
18 | no_token_positional_embeddings: false
19 | share_decoder_input_output_embed: false
20 | character_embeddings: false
21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]"
22 | character_embedding_dim: 4
23 | char_embedder_highway_layers: 2
24 | adaptive_input: false
25 | adaptive_input_factor: 4
26 | adaptive_input_cutoff: null
27 | tie_adaptive_weights: false
28 | tie_adaptive_proj: false
29 | decoder_learned_pos: false
30 | decoder_layerdrop: 0
31 | decoder_layers_to_keep: null
32 | layernorm_embedding: false
33 | no_scale_embedding: false
34 | quant_noise_pq: 0
35 | quant_noise_pq_block_size: 8
36 | quant_noise_scalar: 0
37 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_gpt2_big.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 | activation_fn: "gelu"
3 | dropout: 0.1
4 | attention_dropout: 0.1
5 | activation_dropout: 0.0
6 | relu_dropout: 0.0
7 | decoder_embed_dim: 1600
8 | decoder_output_dim: 1600
9 | decoder_input_dim: 1600
10 | decoder_ffn_embed_dim: 6400
11 | decoder_layers: 48
12 | decoder_attention_heads: 25
13 | decoder_normalize_before: true
14 | no_decoder_final_norm: false
15 | adaptive_softmax_cutoff: null
16 | adaptive_softmax_dropout: 0
17 | adaptive_softmax_factor: 4
18 | no_token_positional_embeddings: false
19 | share_decoder_input_output_embed: false
20 | character_embeddings: false
21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]"
22 | character_embedding_dim: 4
23 | char_embedder_highway_layers: 2
24 | adaptive_input: false
25 | adaptive_input_factor: 4
26 | adaptive_input_cutoff: null
27 | tie_adaptive_weights: false
28 | tie_adaptive_proj: false
29 | decoder_learned_pos: false
30 | decoder_layerdrop: 0
31 | decoder_layers_to_keep: null
32 | layernorm_embedding: false
33 | no_scale_embedding: false
34 | quant_noise_pq: 0
35 | quant_noise_pq_block_size: 8
36 | quant_noise_scalar: 0
37 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_gpt2_medium.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 | activation_fn: "gelu"
3 | dropout: 0.1
4 | attention_dropout: 0.1
5 | activation_dropout: 0.0
6 | relu_dropout: 0.0
7 | decoder_embed_dim: 1280
8 | decoder_output_dim: 1280
9 | decoder_input_dim: 1280
10 | decoder_ffn_embed_dim: 5120
11 | decoder_layers: 36
12 | decoder_attention_heads: 20
13 | decoder_normalize_before: true
14 | no_decoder_final_norm: false
15 | adaptive_softmax_cutoff: null
16 | adaptive_softmax_dropout: 0
17 | adaptive_softmax_factor: 4
18 | no_token_positional_embeddings: false
19 | share_decoder_input_output_embed: false
20 | character_embeddings: false
21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]"
22 | character_embedding_dim: 4
23 | char_embedder_highway_layers: 2
24 | adaptive_input: false
25 | adaptive_input_factor: 4
26 | adaptive_input_cutoff: null
27 | tie_adaptive_weights: false
28 | tie_adaptive_proj: false
29 | decoder_learned_pos: false
30 | decoder_layerdrop: 0
31 | decoder_layers_to_keep: null
32 | layernorm_embedding: false
33 | no_scale_embedding: false
34 | quant_noise_pq: 0
35 | quant_noise_pq_block_size: 8
36 | quant_noise_scalar: 0
37 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/model/transformer_lm/transformer_lm_gpt2_small.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 | activation_fn: "gelu"
3 | dropout: 0.1
4 | attention_dropout: 0.1
5 | activation_dropout: 0.0
6 | relu_dropout: 0.0
7 | decoder_embed_dim: 1024
8 | decoder_output_dim: 1024
9 | decoder_input_dim: 1024
10 | decoder_ffn_embed_dim: 4096
11 | decoder_layers: 24
12 | decoder_attention_heads: 16
13 | decoder_normalize_before: true
14 | no_decoder_final_norm: false
15 | adaptive_softmax_cutoff: null
16 | adaptive_softmax_dropout: 0
17 | adaptive_softmax_factor: 4
18 | no_token_positional_embeddings: false
19 | share_decoder_input_output_embed: false
20 | character_embeddings: false
21 | character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]"
22 | character_embedding_dim: 4
23 | char_embedder_highway_layers: 2
24 | adaptive_input: false
25 | adaptive_input_factor: 4
26 | adaptive_input_cutoff: null
27 | tie_adaptive_weights: false
28 | tie_adaptive_proj: false
29 | decoder_learned_pos: false
30 | decoder_layerdrop: 0
31 | decoder_layers_to_keep: null
32 | layernorm_embedding: false
33 | no_scale_embedding: false
34 | quant_noise_pq: 0
35 | quant_noise_pq_block_size: 8
36 | quant_noise_scalar: 0
37 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/model/wav2vec/vq_wav2vec_gumbel.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 | activation: gelu
3 | vq_type: gumbel
4 | vq_depth: 2
5 | combine_groups: true
6 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/model/wav2vec2/wav2vec2_base.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 |
3 | quantize_targets: true
4 | final_dim: 256
5 | encoder_layerdrop: 0.05
6 | dropout_input: 0.1
7 | dropout_features: 0.1
8 | feature_grad_mult: 0.1
9 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/config/model/wav2vec2/wav2vec2_large.yaml:
--------------------------------------------------------------------------------
1 | # @package _group_
2 |
3 | quantize_targets: true
4 | extractor_mode: layer_norm
5 | layer_norm_first: true
6 | final_dim: 768
7 | latent_temp: [2.0,0.1,0.999995]
8 | encoder_layerdrop: 0.0
9 | dropout_input: 0.0
10 | dropout_features: 0.0
11 | dropout: 0.0
12 | attention_dropout: 0.0
13 | conv_bias: true
14 |
15 | encoder_layers: 24
16 | encoder_embed_dim: 1024
17 | encoder_ffn_embed_dim: 4096
18 | encoder_attention_heads: 16
19 |
20 | feature_grad_mult: 1.0
21 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/criterions/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 | """isort:skip_file"""
6 |
7 | import importlib
8 | import os
9 |
10 | from fairseq import registry
11 | from fairseq.criterions.fairseq_criterion import ( # noqa
12 | FairseqCriterion,
13 | LegacyFairseqCriterion,
14 | )
15 | from omegaconf import DictConfig
16 |
17 |
18 | (
19 | build_criterion_,
20 | register_criterion,
21 | CRITERION_REGISTRY,
22 | CRITERION_DATACLASS_REGISTRY,
23 | ) = registry.setup_registry(
24 | "--criterion", base_class=FairseqCriterion, default="cross_entropy"
25 | )
26 |
27 |
28 | def build_criterion(cfg: DictConfig, task):
29 | return build_criterion_(cfg, task)
30 |
31 |
32 | # automatically import any Python files in the criterions/ directory
33 | for file in sorted(os.listdir(os.path.dirname(__file__))):
34 | if file.endswith(".py") and not file.startswith("_"):
35 | file_name = file[: file.find(".py")]
36 | importlib.import_module("fairseq.criterions." + file_name)
37 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/audio/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/fairseq/data/audio/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/audio/feature_transforms/global_cmvn.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from fairseq.data.audio.feature_transforms import (
3 | AudioFeatureTransform,
4 | register_audio_feature_transform,
5 | )
6 |
7 |
8 | @register_audio_feature_transform("global_cmvn")
9 | class GlobalCMVN(AudioFeatureTransform):
10 | """Global CMVN (cepstral mean and variance normalization). The global mean
11 | and variance need to be pre-computed and stored in NumPy format (.npz)."""
12 |
13 | @classmethod
14 | def from_config_dict(cls, config=None):
15 | _config = {} if config is None else config
16 | return GlobalCMVN(_config.get("stats_npz_path"))
17 |
18 | def __init__(self, stats_npz_path):
19 | self.stats_npz_path = stats_npz_path
20 | stats = np.load(stats_npz_path)
21 | self.mean, self.std = stats["mean"], stats["std"]
22 |
23 | def __repr__(self):
24 | return self.__class__.__name__ + f'(stats_npz_path="{self.stats_npz_path}")'
25 |
26 | def __call__(self, x):
27 | x = np.subtract(x, self.mean)
28 | x = np.divide(x, self.std)
29 | return x
30 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/colorize_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import torch
7 |
8 | from . import BaseWrapperDataset
9 |
10 |
11 | class ColorizeDataset(BaseWrapperDataset):
12 | """ Adds 'colors' property to net input that is obtained from the provided color getter for use by models """
13 |
14 | def __init__(self, dataset, color_getter):
15 | super().__init__(dataset)
16 | self.color_getter = color_getter
17 |
18 | def collater(self, samples):
19 | base_collate = super().collater(samples)
20 | if len(base_collate) > 0:
21 | base_collate["net_input"]["colors"] = torch.tensor(
22 | list(self.color_getter(self.dataset, s["id"]) for s in samples),
23 | dtype=torch.long,
24 | )
25 | return base_collate
26 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/encoders/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 |
7 | import importlib
8 | import os
9 |
10 | from fairseq import registry
11 |
12 |
13 | build_tokenizer, register_tokenizer, TOKENIZER_REGISTRY, _ = registry.setup_registry(
14 | "--tokenizer",
15 | default=None,
16 | )
17 |
18 |
19 | build_bpe, register_bpe, BPE_REGISTRY, _ = registry.setup_registry(
20 | "--bpe",
21 | default=None,
22 | )
23 |
24 |
25 | # automatically import any Python files in the encoders/ directory
26 | for file in sorted(os.listdir(os.path.dirname(__file__))):
27 | if file.endswith(".py") and not file.startswith("_"):
28 | module = file[: file.find(".py")]
29 | importlib.import_module("fairseq.data.encoders." + module)
30 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/encoders/bytes.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 |
7 | from fairseq.data.encoders import register_bpe
8 | from fairseq.data.encoders.byte_utils import (
9 | SPACE,
10 | SPACE_ESCAPE,
11 | byte_encode,
12 | smart_byte_decode,
13 | )
14 |
15 |
16 | @register_bpe("bytes")
17 | class Bytes(object):
18 | def __init__(self, *unused):
19 | pass
20 |
21 | @staticmethod
22 | def add_args(parser):
23 | pass
24 |
25 | @staticmethod
26 | def encode(x: str) -> str:
27 | encoded = byte_encode(x)
28 | escaped = encoded.replace(SPACE, SPACE_ESCAPE)
29 | return SPACE.join(list(escaped))
30 |
31 | @staticmethod
32 | def decode(x: str) -> str:
33 | unescaped = x.replace(SPACE, "").replace(SPACE_ESCAPE, SPACE)
34 | return smart_byte_decode(unescaped)
35 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/encoders/characters.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 |
7 | from fairseq.data.encoders import register_bpe
8 |
9 |
10 | SPACE = chr(32)
11 | SPACE_ESCAPE = chr(9601)
12 |
13 |
14 | @register_bpe("characters")
15 | class Characters(object):
16 | def __init__(self, *unused):
17 | pass
18 |
19 | @staticmethod
20 | def add_args(parser):
21 | pass
22 |
23 | @staticmethod
24 | def encode(x: str) -> str:
25 | escaped = x.replace(SPACE, SPACE_ESCAPE)
26 | return SPACE.join(list(escaped))
27 |
28 | @staticmethod
29 | def decode(x: str) -> str:
30 | return x.replace(SPACE, "").replace(SPACE_ESCAPE, SPACE)
31 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/encoders/nltk_tokenizer.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from fairseq.data.encoders import register_tokenizer
7 | from fairseq.dataclass import FairseqDataclass
8 |
9 |
10 | @register_tokenizer("nltk", dataclass=FairseqDataclass)
11 | class NLTKTokenizer(object):
12 | def __init__(self, *unused):
13 | try:
14 | from nltk.tokenize import word_tokenize
15 |
16 | self.word_tokenize = word_tokenize
17 | except ImportError:
18 | raise ImportError("Please install nltk with: pip install nltk")
19 |
20 | def encode(self, x: str) -> str:
21 | return " ".join(self.word_tokenize(x))
22 |
23 | def decode(self, x: str) -> str:
24 | return x
25 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/encoders/space_tokenizer.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import re
7 |
8 | from fairseq.data.encoders import register_tokenizer
9 | from fairseq.dataclass import FairseqDataclass
10 |
11 |
12 | @register_tokenizer("space", dataclass=FairseqDataclass)
13 | class SpaceTokenizer(object):
14 | def __init__(self, *unused):
15 | self.space_tok = re.compile(r"\s+")
16 |
17 | def encode(self, x: str) -> str:
18 | return self.space_tok.sub(" ", x)
19 |
20 | def decode(self, x: str) -> str:
21 | return x
22 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/encoders/utils.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import torch
7 | from fairseq.data import encoders
8 |
9 |
10 | def get_whole_word_mask(args, dictionary):
11 | bpe = encoders.build_bpe(args)
12 | if bpe is not None:
13 |
14 | def is_beginning_of_word(i):
15 | if i < dictionary.nspecial:
16 | # special elements are always considered beginnings
17 | return True
18 | tok = dictionary[i]
19 | if tok.startswith("madeupword"):
20 | return True
21 | try:
22 | return bpe.is_beginning_of_word(tok)
23 | except ValueError:
24 | return True
25 |
26 | mask_whole_words = torch.ByteTensor(
27 | list(map(is_beginning_of_word, range(len(dictionary))))
28 | )
29 | return mask_whole_words
30 | return None
31 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/id_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import torch
7 |
8 | from . import FairseqDataset
9 |
10 |
11 | class IdDataset(FairseqDataset):
12 | def __getitem__(self, index):
13 | return index
14 |
15 | def __len__(self):
16 | return 0
17 |
18 | def collater(self, samples):
19 | return torch.tensor(samples)
20 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/legacy/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .block_pair_dataset import BlockPairDataset
7 | from .masked_lm_dataset import MaskedLMDataset
8 | from .masked_lm_dictionary import BertDictionary, MaskedLMDictionary
9 |
10 |
11 | __all__ = [
12 | "BertDictionary",
13 | "BlockPairDataset",
14 | "MaskedLMDataset",
15 | "MaskedLMDictionary",
16 | ]
17 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/list_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import BaseWrapperDataset
7 |
8 |
9 | class ListDataset(BaseWrapperDataset):
10 | def __init__(self, dataset, sizes=None):
11 | super().__init__(dataset)
12 | self._sizes = sizes
13 |
14 | def __iter__(self):
15 | for x in self.dataset:
16 | yield x
17 |
18 | def collater(self, samples):
19 | return samples
20 |
21 | @property
22 | def sizes(self):
23 | return self._sizes
24 |
25 | def num_tokens(self, index):
26 | return self.sizes[index]
27 |
28 | def size(self, index):
29 | return self.sizes[index]
30 |
31 | def set_epoch(self, epoch):
32 | pass
33 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/lru_cache_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from functools import lru_cache
7 |
8 | from . import BaseWrapperDataset
9 |
10 |
11 | class LRUCacheDataset(BaseWrapperDataset):
12 | def __init__(self, dataset, token=None):
13 | super().__init__(dataset)
14 |
15 | @lru_cache(maxsize=8)
16 | def __getitem__(self, index):
17 | return self.dataset[index]
18 |
19 | @lru_cache(maxsize=8)
20 | def collater(self, samples):
21 | return self.dataset.collater(samples)
22 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/multilingual/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/num_samples_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import FairseqDataset
7 |
8 |
9 | class NumSamplesDataset(FairseqDataset):
10 | def __getitem__(self, index):
11 | return 1
12 |
13 | def __len__(self):
14 | return 0
15 |
16 | def collater(self, samples):
17 | return sum(samples)
18 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/numel_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import numpy as np
7 | import torch
8 |
9 | from . import BaseWrapperDataset
10 |
11 |
12 | class NumelDataset(BaseWrapperDataset):
13 | def __init__(self, dataset, reduce=False):
14 | super().__init__(dataset)
15 | self.reduce = reduce
16 |
17 | def __getitem__(self, index):
18 | item = self.dataset[index]
19 | if torch.is_tensor(item):
20 | return torch.numel(item)
21 | else:
22 | return np.size(item)
23 |
24 | def __len__(self):
25 | return len(self.dataset)
26 |
27 | def collater(self, samples):
28 | if self.reduce:
29 | return sum(samples)
30 | else:
31 | return torch.tensor(samples)
32 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/offset_tokens_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import BaseWrapperDataset
7 |
8 |
9 | class OffsetTokensDataset(BaseWrapperDataset):
10 | def __init__(self, dataset, offset):
11 | super().__init__(dataset)
12 | self.offset = offset
13 |
14 | def __getitem__(self, idx):
15 | return self.dataset[idx] + self.offset
16 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/pad_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from fairseq.data import data_utils
7 |
8 | from . import BaseWrapperDataset
9 |
10 |
11 | class PadDataset(BaseWrapperDataset):
12 | def __init__(self, dataset, pad_idx, left_pad):
13 | super().__init__(dataset)
14 | self.pad_idx = pad_idx
15 | self.left_pad = left_pad
16 |
17 | def collater(self, samples):
18 | return data_utils.collate_tokens(samples, self.pad_idx, left_pad=self.left_pad)
19 |
20 |
21 | class LeftPadDataset(PadDataset):
22 | def __init__(self, dataset, pad_idx):
23 | super().__init__(dataset, pad_idx, left_pad=True)
24 |
25 |
26 | class RightPadDataset(PadDataset):
27 | def __init__(self, dataset, pad_idx):
28 | super().__init__(dataset, pad_idx, left_pad=False)
29 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/prepend_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import numpy as np
7 | import torch
8 |
9 | from . import BaseWrapperDataset
10 |
11 |
12 | class PrependDataset(BaseWrapperDataset):
13 | def __init__(self, dataset, prepend_getter, ensure_first_token_is=None):
14 | super().__init__(dataset)
15 | self.prepend_getter = prepend_getter
16 | self.ensure_first_token = ensure_first_token_is
17 |
18 | def __getitem__(self, idx):
19 | item = self.dataset[idx]
20 | is_tuple = isinstance(item, tuple)
21 | src = item[0] if is_tuple else item
22 |
23 | assert self.ensure_first_token is None or src[0] == self.ensure_first_token
24 | prepend_idx = self.prepend_getter(self.dataset, idx)
25 | assert isinstance(prepend_idx, int)
26 | src[0] = prepend_idx
27 | item = tuple((src,) + item[1:]) if is_tuple else src
28 | return item
29 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/raw_label_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import torch
7 |
8 | from . import FairseqDataset
9 |
10 |
11 | class RawLabelDataset(FairseqDataset):
12 | def __init__(self, labels):
13 | super().__init__()
14 | self.labels = labels
15 |
16 | def __getitem__(self, index):
17 | return self.labels[index]
18 |
19 | def __len__(self):
20 | return len(self.labels)
21 |
22 | def collater(self, samples):
23 | return torch.tensor(samples)
24 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/roll_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import torch
7 |
8 | from . import BaseWrapperDataset
9 |
10 |
11 | class RollDataset(BaseWrapperDataset):
12 | def __init__(self, dataset, shifts):
13 | super().__init__(dataset)
14 | self.shifts = shifts
15 |
16 | def __getitem__(self, index):
17 | item = self.dataset[index]
18 | return torch.roll(item, self.shifts)
19 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/sort_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import numpy as np
7 |
8 | from . import BaseWrapperDataset
9 |
10 |
11 | class SortDataset(BaseWrapperDataset):
12 | def __init__(self, dataset, sort_order):
13 | super().__init__(dataset)
14 | if not isinstance(sort_order, (list, tuple)):
15 | sort_order = [sort_order]
16 | self.sort_order = sort_order
17 |
18 | assert all(len(so) == len(dataset) for so in sort_order)
19 |
20 | def ordered_indices(self):
21 | return np.lexsort(self.sort_order)
22 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/data/strip_token_dataset.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import BaseWrapperDataset
7 |
8 |
9 | class StripTokenDataset(BaseWrapperDataset):
10 | def __init__(self, dataset, id_to_strip):
11 | super().__init__(dataset)
12 | self.id_to_strip = id_to_strip
13 |
14 | def __getitem__(self, index):
15 | item = self.dataset[index]
16 | while len(item) > 0 and item[-1] == self.id_to_strip:
17 | item = item[:-1]
18 | while len(item) > 0 and item[0] == self.id_to_strip:
19 | item = item[1:]
20 | return item
21 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/dataclass/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .configs import FairseqDataclass
7 | from .constants import ChoiceEnum
8 |
9 |
10 | __all__ = [
11 | "FairseqDataclass",
12 | "ChoiceEnum",
13 | ]
14 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/distributed/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .distributed_timeout_wrapper import DistributedTimeoutWrapper
7 | from .fully_sharded_data_parallel import fsdp_enable_wrap, fsdp_wrap, FullyShardedDataParallel
8 | from .legacy_distributed_data_parallel import LegacyDistributedDataParallel
9 | from .module_proxy_wrapper import ModuleProxyWrapper
10 | from .tpu_distributed_data_parallel import TPUDistributedDataParallel
11 |
12 |
13 | __all__ = [
14 | "DistributedTimeoutWrapper",
15 | "fsdp_enable_wrap",
16 | "fsdp_wrap",
17 | "FullyShardedDataParallel",
18 | "LegacyDistributedDataParallel",
19 | "ModuleProxyWrapper",
20 | "TPUDistributedDataParallel",
21 | ]
22 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/logging/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/fairseq/logging/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/model_parallel/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from . import criterions, models, modules # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/model_parallel/criterions/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import importlib
7 | import os
8 |
9 |
10 | # automatically import any Python files in the criterions/ directory
11 | for file in sorted(os.listdir(os.path.dirname(__file__))):
12 | if file.endswith(".py") and not file.startswith("_"):
13 | module = file[: file.find(".py")]
14 | importlib.import_module("fairseq.model_parallel.criterions." + module)
15 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/model_parallel/models/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import importlib
7 | import os
8 |
9 |
10 | # automatically import any Python files in the models/ directory
11 | models_dir = os.path.dirname(__file__)
12 | for file in os.listdir(models_dir):
13 | path = os.path.join(models_dir, file)
14 | if (
15 | not file.startswith("_")
16 | and not file.startswith(".")
17 | and (file.endswith(".py") or os.path.isdir(path))
18 | ):
19 | model_name = file[: file.find(".py")] if file.endswith(".py") else file
20 | module = importlib.import_module("fairseq.model_parallel.models." + model_name)
21 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/model_parallel/models/pipeline_parallel_transformer/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .model import * # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/model_parallel/models/roberta/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .model import * # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/model_parallel/modules/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 | """isort:skip_file"""
6 |
7 | from .multihead_attention import ModelParallelMultiheadAttention
8 | from .transformer_layer import (
9 | ModelParallelTransformerEncoderLayer,
10 | ModelParallelTransformerDecoderLayer,
11 | )
12 |
13 | __all__ = [
14 | "ModelParallelMultiheadAttention",
15 | "ModelParallelTransformerEncoderLayer",
16 | "ModelParallelTransformerDecoderLayer",
17 | ]
18 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/models/bart/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .hub_interface import * # noqa
7 | from .model import * # noqa
8 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/models/hubert/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .hubert import * # noqa
7 | from .hubert_asr import * # noqa
8 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/models/huggingface/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import importlib
7 | import os
8 |
9 |
10 | # automatically import any Python files in the models/huggingface/ directory
11 | models_dir = os.path.dirname(__file__)
12 | for file in os.listdir(models_dir):
13 | path = os.path.join(models_dir, file)
14 | if (
15 | not file.startswith("_")
16 | and not file.startswith(".")
17 | and (file.endswith(".py") or os.path.isdir(path))
18 | ):
19 | model_name = file[: file.find(".py")] if file.endswith(".py") else file
20 | module = importlib.import_module("fairseq.models.huggingface." + model_name)
21 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/models/nat/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 | """isort:skip_file"""
6 |
7 | from .fairseq_nat_model import *
8 | from .nonautoregressive_transformer import *
9 | from .nat_crf_transformer import *
10 | from .iterative_nonautoregressive_transformer import *
11 | from .cmlm_transformer import *
12 | from .levenshtein_transformer import *
13 | from .insertion_transformer import *
14 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/models/roberta/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .hub_interface import * # noqa
7 | from .model import * # noqa
8 | from .enc_dec import * # noqa
9 | from .model_camembert import * # noqa
10 | from .model_gottbert import * # noqa
11 | from .model_xlmr import * # noqa
12 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/models/speech_to_text/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .berard import * # noqa
7 | from .convtransformer import * # noqa
8 | from .s2t_transformer import * # noqa
9 | from .xm_transformer import * # noqa
10 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/models/wav2vec/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .wav2vec import * # noqa
7 | from .wav2vec2 import * # noqa
8 | from .wav2vec2_asr import * # noqa
9 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/dynamicconv_layer/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .dynamicconv_layer import DynamicconvLayer # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/dynamicconv_layer/dynamiconv_cpu.cpp:
--------------------------------------------------------------------------------
1 | #include
2 | #include
3 |
4 | std::vector
5 | dynamicconv_cpu_forward(float* input, float* filters, int padding_l);
6 |
7 | std::vector dynamicconv_cpu_backward(
8 | float* gradOutput,
9 | int padding_l,
10 | float* input,
11 | float* filters);
12 |
13 | std::vector
14 | dynamicconv_forward(float* input, float* filters, int padding_l) {
15 | return dynamicconv_cpu_forward(input, filters, padding_l);
16 | }
17 |
18 | std::vector dynamicconv_backward(
19 | float* gradOutput,
20 | int padding_l,
21 | float* input,
22 | float* filters) {
23 | return dynamicconv_cpu_backward(gradOutput, padding_l, input, filters);
24 | }
25 |
26 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
27 | m.def("forward", &dynamicconv_forward, "dynamicconv forward (CPU)");
28 | m.def("backward", &dynamicconv_backward, "dynamicconv backward (CPU)");
29 | }
30 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/dynamicconv_layer/setup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | from setuptools import setup
8 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension
9 |
10 |
11 | setup(
12 | name="dynamicconv_layer",
13 | ext_modules=[
14 | CUDAExtension(
15 | name="dynamicconv_cuda",
16 | sources=[
17 | "dynamicconv_cuda.cpp",
18 | "dynamicconv_cuda_kernel.cu",
19 | ],
20 | ),
21 | ],
22 | cmdclass={"build_ext": BuildExtension},
23 | )
24 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/fp32_group_norm.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 | """
6 | Layer norm done in fp32 (for fp16 training)
7 | """
8 |
9 | import torch.nn as nn
10 | import torch.nn.functional as F
11 |
12 |
13 | class Fp32GroupNorm(nn.GroupNorm):
14 | def __init__(self, *args, **kwargs):
15 | super().__init__(*args, **kwargs)
16 |
17 | def forward(self, input):
18 | output = F.group_norm(
19 | input.float(),
20 | self.num_groups,
21 | self.weight.float() if self.weight is not None else None,
22 | self.bias.float() if self.bias is not None else None,
23 | self.eps,
24 | )
25 | return output.type_as(input)
26 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/gelu.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 | """
6 | See "Gaussian Error Linear Units (GELUs)" by Dan Hendrycks and Kevin Gimpel with
7 | the corresponding GitHub repo: https://github.com/hendrycks/GELUs
8 | """
9 |
10 | import math
11 |
12 | import torch
13 | import torch.nn as nn
14 |
15 |
16 | def gelu_accurate(x):
17 | if not hasattr(gelu_accurate, "_a"):
18 | gelu_accurate._a = math.sqrt(2 / math.pi)
19 | return (
20 | 0.5 * x * (1 + torch.tanh(gelu_accurate._a * (x + 0.044715 * torch.pow(x, 3))))
21 | )
22 |
23 |
24 | def gelu(x: torch.Tensor) -> torch.Tensor:
25 | return torch.nn.functional.gelu(x.float()).type_as(x)
26 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/grad_multiply.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import torch
7 |
8 |
9 | class GradMultiply(torch.autograd.Function):
10 | @staticmethod
11 | def forward(ctx, x, scale):
12 | ctx.scale = scale
13 | res = x.new(x)
14 | return res
15 |
16 | @staticmethod
17 | def backward(ctx, grad):
18 | return grad * ctx.scale, None
19 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/lightconv_layer/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .lightconv_layer import LightconvLayer # noqa
7 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/lightconv_layer/setup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | from setuptools import setup
8 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension
9 |
10 |
11 | setup(
12 | name="lightconv_layer",
13 | ext_modules=[
14 | CUDAExtension(
15 | "lightconv_cuda",
16 | [
17 | "lightconv_cuda.cpp",
18 | "lightconv_cuda_kernel.cu",
19 | ],
20 | ),
21 | ],
22 | cmdclass={"build_ext": BuildExtension},
23 | )
24 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/quantization/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/fairseq/modules/quantization/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/quantization/pq/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .utils import SizeTracker, get_param, attrsetter, quantize_model_ # NOQA
7 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/quantization/pq/modules/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .qconv import PQConv2d # NOQA
7 | from .qemb import PQEmbedding # NOQA
8 | from .qlinear import PQLinear # NOQA
9 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/quantization/scalar/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .utils import quantize_model_ # NOQA
7 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/quantization/scalar/modules/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from .qact import ActivationQuantizer # NOQA
7 | from .qconv import IntConv2d # NOQA
8 | from .qemb import IntEmbedding # NOQA
9 | from .qlinear import IntLinear # NOQA
10 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/same_pad.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 |
7 | from torch import nn
8 |
9 |
10 | class SamePad(nn.Module):
11 | def __init__(self, kernel_size, causal=False):
12 | super().__init__()
13 | if causal:
14 | self.remove = kernel_size - 1
15 | else:
16 | self.remove = 1 if kernel_size % 2 == 0 else 0
17 |
18 | def forward(self, x):
19 | if self.remove > 0:
20 | x = x[:, :, : -self.remove]
21 | return x
22 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/scalar_bias.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 | #
6 |
7 | import torch
8 |
9 |
10 | class ScalarBias(torch.autograd.Function):
11 | """
12 | Adds a vector of scalars, used in self-attention mechanism to allow
13 | the model to optionally attend to this vector instead of the past
14 | """
15 |
16 | @staticmethod
17 | def forward(ctx, input, dim, bias_init):
18 | size = list(input.size())
19 | size[dim] += 1
20 | output = input.new(*size).fill_(bias_init)
21 | output.narrow(dim, 1, size[dim] - 1).copy_(input)
22 | ctx.dim = dim
23 | return output
24 |
25 | @staticmethod
26 | def backward(ctx, grad):
27 | return grad.narrow(ctx.dim, 1, grad.size(ctx.dim) - 1), None, None
28 |
29 |
30 | def scalar_bias(input, dim, bias_init=0):
31 | return ScalarBias.apply(input, dim, bias_init)
32 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/transpose_last.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 | """
6 | transpose last 2 dimensions of the input
7 | """
8 |
9 | import torch.nn as nn
10 |
11 |
12 | class TransposeLast(nn.Module):
13 | def __init__(self, deconstruct_idx=None):
14 | super().__init__()
15 | self.deconstruct_idx = deconstruct_idx
16 |
17 | def forward(self, x):
18 | if self.deconstruct_idx is not None:
19 | x = x[self.deconstruct_idx]
20 | return x.transpose(-2, -1)
21 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/modules/unfold.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import torch.nn.functional as F
7 |
8 |
9 | def unfold1d(x, kernel_size, padding_l, pad_value=0):
10 | """unfold T x B x C to T x B x C x K"""
11 | if kernel_size > 1:
12 | T, B, C = x.size()
13 | x = F.pad(
14 | x, (0, 0, 0, 0, padding_l, kernel_size - 1 - padding_l), value=pad_value
15 | )
16 | x = x.as_strided((T, B, C, kernel_size), (B * C, C, 1, B * C))
17 | else:
18 | x = x.unsqueeze(3)
19 | return x
20 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/scoring/chrf.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | from fairseq.scoring import BaseScorer, register_scorer
7 |
8 |
9 | @register_scorer("chrf")
10 | class ChrFScorer(BaseScorer):
11 | def __init__(self, args):
12 | super(ChrFScorer, self).__init__(args)
13 | import sacrebleu
14 |
15 | self.sacrebleu = sacrebleu
16 |
17 | def add_string(self, ref, pred):
18 | self.ref.append(ref)
19 | self.pred.append(pred)
20 |
21 | def score(self, order=4):
22 | return self.result_string(order).score
23 |
24 | def result_string(self, order=4):
25 | if order != 4:
26 | raise NotImplementedError
27 | return self.sacrebleu.corpus_chrf(self.pred, [self.ref]).format()
28 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/tokenizer.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import re
7 |
8 |
9 | SPACE_NORMALIZER = re.compile(r"\s+")
10 |
11 |
12 | def tokenize_line(line):
13 | line = SPACE_NORMALIZER.sub(" ", line)
14 | line = line.strip()
15 | return line.split()
16 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq/version.txt:
--------------------------------------------------------------------------------
1 | 1.0.0a0
2 |
--------------------------------------------------------------------------------
/Machine_Translation/fairseq_cli/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/fairseq_cli/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/pyproject.toml:
--------------------------------------------------------------------------------
1 | [build-system]
2 | requires = ["setuptools", "wheel", "cython"]
3 | build-backend = "setuptools.build_meta"
4 |
--------------------------------------------------------------------------------
/Machine_Translation/scripts/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/scripts/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/scripts/compound_split_bleu.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | if [ $# -ne 1 ]; then
4 | echo "usage: $0 GENERATE_PY_OUTPUT"
5 | exit 1
6 | fi
7 |
8 | GEN=$1
9 |
10 | SYS=$GEN.sys
11 | REF=$GEN.ref
12 |
13 | if [ $(tail -n 1 $GEN | grep BLEU | wc -l) -ne 1 ]; then
14 | echo "not done generating"
15 | exit
16 | fi
17 |
18 | grep ^H $GEN | awk -F '\t' '{print $NF}' | perl -ple 's{(\S)-(\S)}{$1 ##AT##-##AT## $2}g' > $SYS
19 | grep ^T $GEN | cut -f2- | perl -ple 's{(\S)-(\S)}{$1 ##AT##-##AT## $2}g' > $REF
20 | fairseq-score --sys $SYS --ref $REF
21 |
--------------------------------------------------------------------------------
/Machine_Translation/scripts/constraints/validate.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | #
3 | # Copyright (c) Facebook, Inc. and its affiliates.
4 | #
5 | # This source code is licensed under the MIT license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 | import sys
9 |
10 |
11 | """Reads in a fairseq output file, and verifies that the constraints
12 | (C- lines) are present in the output (the first H- line). Assumes that
13 | constraints are listed prior to the first hypothesis.
14 | """
15 |
16 | constraints = []
17 | found = 0
18 | total = 0
19 | for line in sys.stdin:
20 | if line.startswith("C-"):
21 | constraints.append(line.rstrip().split("\t")[1])
22 | elif line.startswith("H-"):
23 | text = line.split("\t")[2]
24 |
25 | for constraint in constraints:
26 | total += 1
27 | if constraint in text:
28 | found += 1
29 | else:
30 | print(f"No {constraint} in {text}", file=sys.stderr)
31 |
32 | constraints = []
33 |
34 | print(f"Found {found} / {total} = {100 * found / total:.1f}%")
35 |
--------------------------------------------------------------------------------
/Machine_Translation/scripts/convert_dictionary.lua:
--------------------------------------------------------------------------------
1 | -- Copyright (c) Facebook, Inc. and its affiliates.
2 | --
3 | -- This source code is licensed under the MIT license found in the
4 | -- LICENSE file in the root directory of this source tree.
5 | --
6 | -- Usage: convert_dictionary.lua
7 | require 'fairseq'
8 | require 'torch'
9 | require 'paths'
10 |
11 | if #arg < 1 then
12 | print('usage: convert_dictionary.lua ')
13 | os.exit(1)
14 | end
15 | if not paths.filep(arg[1]) then
16 | print('error: file does not exit: ' .. arg[1])
17 | os.exit(1)
18 | end
19 |
20 | dict = torch.load(arg[1])
21 | dst = paths.basename(arg[1]):gsub('.th7', '.txt')
22 | assert(dst:match('.txt$'))
23 |
24 | f = io.open(dst, 'w')
25 | for idx, symbol in ipairs(dict.index_to_symbol) do
26 | if idx > dict.cutoff then
27 | break
28 | end
29 | f:write(symbol)
30 | f:write(' ')
31 | f:write(dict.index_to_freq[idx])
32 | f:write('\n')
33 | end
34 | f:close()
35 |
--------------------------------------------------------------------------------
/Machine_Translation/scripts/sacrebleu.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | if [ $# -ne 4 ]; then
4 | echo "usage: $0 TESTSET SRCLANG TGTLANG GEN"
5 | exit 1
6 | fi
7 |
8 | TESTSET=$1
9 | SRCLANG=$2
10 | TGTLANG=$3
11 |
12 | GEN=$4
13 |
14 | if ! command -v sacremoses &> /dev/null
15 | then
16 | echo "sacremoses could not be found, please install with: pip install sacremoses"
17 | exit
18 | fi
19 |
20 | grep ^H $GEN \
21 | | sed 's/^H\-//' \
22 | | sort -n -k 1 \
23 | | cut -f 3 \
24 | | sacremoses detokenize \
25 | > $GEN.sorted.detok
26 |
27 | sacrebleu --test-set $TESTSET --language-pair "${SRCLANG}-${TGTLANG}" < $GEN.sorted.detok
28 |
--------------------------------------------------------------------------------
/Machine_Translation/scripts/spm_train.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | # All rights reserved.
4 | #
5 | # This source code is licensed under the license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 | from __future__ import absolute_import, division, print_function, unicode_literals
9 |
10 | import sys
11 |
12 | import sentencepiece as spm
13 |
14 |
15 | if __name__ == "__main__":
16 | spm.SentencePieceTrainer.Train(" ".join(sys.argv[1:]))
17 |
--------------------------------------------------------------------------------
/Machine_Translation/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/tests/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/tests/distributed/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/tests/distributed/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/tests/gpu/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/tests/gpu/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/tests/gpu/transformer_quantization_config.yaml:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | # This file defines example configuration arguments for quantizing
7 | # a transformer model with product quantization
8 |
9 | n_centroids:
10 | Linear:
11 | key: in_features
12 | value: {"*": 8}
13 | Embedding:
14 | key: embedding_dim
15 | value: {"*": 8}
16 |
17 | block_sizes:
18 | Linear:
19 | key: fuzzy_name
20 | value: {fc: 8, attn: 4, emb: 4}
21 | Embedding:
22 | key: fuzzy_name
23 | value: {emb: 8}
24 |
25 | layers_to_quantize:
26 | - decoder\\.layers\\.\d+\\.fc[12]
27 | - decoder\\.embed_tokens\\.embeddings\\.[012]\\.[01]
28 | - decoder\\.layers\\.\d+\\.self_attn\\.(k_proj|v_proj|q_proj|out_proj)
29 |
--------------------------------------------------------------------------------
/Machine_Translation/tests/speech_recognition/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NYCU-MLLab/Contrastive-Disentangled-Memory-for-sequential-Learning/a075f8cd73374e1aa548dd28e4c47543380f7829/Machine_Translation/tests/speech_recognition/__init__.py
--------------------------------------------------------------------------------
/Machine_Translation/tests/test_iopath.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 |
6 | import unittest
7 | from unittest import mock
8 |
9 |
10 | class TestIOPath(unittest.TestCase):
11 |
12 | def test_no_iopath(self):
13 | from .test_reproducibility import TestReproducibility
14 |
15 | with mock.patch.dict("sys.modules", {"iopath": None}):
16 | # reuse reproducibility tests, which are e2e tests that should cover
17 | # most checkpoint related functionality
18 | TestReproducibility._test_reproducibility(self, "test_reproducibility")
19 |
20 | def test_no_supports_rename(self):
21 | from .test_reproducibility import TestReproducibility
22 |
23 | with mock.patch("fairseq.file_io.PathManager.supports_rename") as mock_fn:
24 | mock_fn.return_value = False
25 | TestReproducibility._test_reproducibility(self, "test_reproducibility")
26 |
27 |
28 | if __name__ == "__main__":
29 | unittest.main()
30 |
--------------------------------------------------------------------------------
/Machine_Translation/train.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3 -u
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | #
4 | # This source code is licensed under the MIT license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | """
7 | Legacy entry point. Use fairseq_cli/train.py or fairseq-train instead.
8 | """
9 |
10 | from fairseq_cli.train import cli_main
11 |
12 |
13 | if __name__ == "__main__":
14 | cli_main()
15 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Contrastive-Disentangled-Memory-for-sequential-Learning
2 |
3 | # Introduction
4 |
5 | This is a source code for Transformer with a contrastive disentangled memory.
6 | The implementation includes machine translation and online speech recognition.
7 |
8 |
9 | ## Requirements and Frameworks
10 | - Python 3.6.1+
11 | - Fairseq
12 | - ESPnet
13 |
14 | ### To do machine translation
15 | ```sh
16 | cd Machine_Translation
17 | ```
18 |
19 | ### To do speech recognition
20 | ```sh
21 | cd Speech_Recognition
22 | ```
23 |
--------------------------------------------------------------------------------
/Speech_Recognition/README.md:
--------------------------------------------------------------------------------
1 | ## Python Requirement
2 |
3 | ```sh
4 | torch
5 | chainer==6.0.0 cupy==6.0.0
6 | torchaudio
7 | torch_optimizer
8 | ```
9 |
10 | ## Setup
11 |
12 | ```sh
13 | cd tools
14 | make
15 | ```
16 |
17 | ## Download Aishell dadaset and Preprocessing
18 |
19 | ```sh
20 | cd egs/aishell/asr
21 | ./run.sh --stage -1
22 | ```
23 |
24 | ## Train
25 |
26 | ```sh
27 | cd egs/aishell/asr
28 | ./run.sh --stage 4
29 | ```
30 |
31 | ## Test
32 |
33 | ```sh
34 | cd egs/aishell/asr
35 | ./run.sh --stage 5
36 | ```
37 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/RESULTS.md:
--------------------------------------------------------------------------------
1 | # Result
2 | - [Online compressive transformer version 1](https://drive.google.com/drive/folders/1wmG7B2Ld0k8B9aYqYJU7lhS9DiwNqNKq?usp=sharing)
3 | - [Online compressive transformer version 2](https://drive.google.com/drive/folders/1Ex2qypmdnzLJDzH8LyXzbbfyuq4yVrhN?usp=sharing)
4 | - [Synchronous transformer[1]](https://drive.google.com/drive/folders/1_OyG2p5EikSRP_T37odkH0-EHIdpDG-F?usp=sharing)
5 | - [Transformer](https://drive.google.com/drive/folders/1pVspTH4ljY4ddYujlSo4PRKjDICC1reV?usp=sharing)
6 |
7 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/decode.yaml:
--------------------------------------------------------------------------------
1 | batchsize: 0
2 | beam-size: 5
3 | penalty: 0.0
4 | maxlenratio: 0.0
5 | minlenratio: 0.0
6 | ctc-weight: 0.0
7 | #lm-weight: 0.7
8 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/decode_memory_transformer.yaml:
--------------------------------------------------------------------------------
1 | # decoding parameters
2 | batch: 0
3 | score-norm-transducer: true
4 | beam-size: 5
5 | ctc-weight: 0
6 | # lm-weight: 0.05
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/decode_sync_transformer.yaml:
--------------------------------------------------------------------------------
1 | # decoding parameters
2 | batch: 0
3 | score-norm-transducer: true
4 | beam-size: 5
5 | # lm-weight: 0.05
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/fbank.conf:
--------------------------------------------------------------------------------
1 | --sample-frequency=16000
2 | --num-mel-bins=80
3 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/gpu.conf:
--------------------------------------------------------------------------------
1 | # Default configuration
2 | command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64*
3 | option mem=* -l mem_free=$0,ram_free=$0
4 | option mem=0 # Do not add anything to qsub_opts
5 | option num_threads=* -pe smp $0
6 | option num_threads=1 # Do not add anything to qsub_opts
7 | option max_jobs_run=* -tc $0
8 | default gpu=0
9 | option gpu=0
10 | option gpu=* -l 'hostname=b1[12345678]*|c*,gpu=$0' -q g.q
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/lm.yaml:
--------------------------------------------------------------------------------
1 | # rnnlm related
2 | layer: 2
3 | unit: 650
4 | opt: sgd # or adam
5 | batchsize: 64 # batch size in LM training
6 | epoch: 20 # if the data size is large, we can reduce this
7 | patience: 3
8 | maxlen: 100 # if sentence length > lm_maxlen, lm_batchsize is automatically reduced
9 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/pitch.conf:
--------------------------------------------------------------------------------
1 | --sample-frequency=16000
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/queue.conf:
--------------------------------------------------------------------------------
1 | # Default configuration
2 | command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64*
3 | option mem=* -l mem_free=$0,ram_free=$0
4 | option mem=0 # Do not add anything to qsub_opts
5 | option num_threads=* -pe smp $0
6 | option num_threads=1 # Do not add anything to qsub_opts
7 | option max_jobs_run=* -tc $0
8 | default gpu=0
9 | option gpu=0
10 | option gpu=* -l gpu=$0 -q g.q
11 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/slurm.conf:
--------------------------------------------------------------------------------
1 | # Default configuration
2 | command sbatch --export=PATH --ntasks-per-node=1
3 | option time=* --time $0
4 | option mem=* --mem-per-cpu $0
5 | option mem=0 # Do not add anything to qsub_opts
6 | option num_threads=* --cpus-per-task $0 --ntasks-per-node=1
7 | option num_threads=1 --cpus-per-task 1 --ntasks-per-node=1 # Do not add anything to qsub_opts
8 | default gpu=0
9 | option gpu=0 -p cpu
10 | option gpu=* -p gpu --gres=gpu:$0
11 | # note: the --max-jobs-run option is supported as a special case
12 | # by slurm.pl and you don't have to handle it in the config file.
13 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/specaug.yaml:
--------------------------------------------------------------------------------
1 | process:
2 | # these three processes are a.k.a. SpecAugument
3 | - type: "time_warp"
4 | max_time_warp: 5
5 | inplace: true
6 | mode: "PIL"
7 | - type: "freq_mask"
8 | F: 30
9 | n_mask: 2
10 | inplace: true
11 | replace_with_zero: false
12 | - type: "time_mask"
13 | T: 40
14 | n_mask: 2
15 | inplace: true
16 | replace_with_zero: false
17 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/train.yaml:
--------------------------------------------------------------------------------
1 | # network architecture
2 | # encoder related
3 | elayers: 12
4 | eunits: 2048
5 | # decoder related
6 | dlayers: 6
7 | dunits: 2048
8 | # attention related
9 | adim: 256
10 | aheads: 4
11 |
12 | # hybrid CTC/attention
13 | mtlalpha: 0.3
14 |
15 | # label smoothing
16 | lsm-weight: 0.1
17 |
18 | # minibatch related
19 | batch-size: 32
20 | maxlen-in: 512 # if input length > maxlen-in, batchsize is automatically reduced
21 | maxlen-out: 150 # if output length > maxlen-out, batchsize is automatically reduced
22 |
23 | # optimization related
24 | sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
25 | opt: noam
26 | accum-grad: 2
27 | grad-clip: 5
28 | patience: 0
29 | epochs: 50
30 | dropout-rate: 0.1
31 |
32 | # transformer specific setting
33 | backend: pytorch
34 | model-module: "espnet.nets.pytorch_backend.e2e_asr_transformer:E2E"
35 | transformer-input-layer: conv2d # encoder architecture type
36 | transformer-lr: 1.0
37 | transformer-warmup-steps: 25000
38 | transformer-attn-dropout-rate: 0.0
39 | transformer-length-normalized-loss: false
40 | transformer-init: pytorch
41 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/train_transducer.yaml:
--------------------------------------------------------------------------------
1 | # minibatch related
2 | batch-size: 20
3 | maxlen-in: 800
4 | maxlen-out: 150
5 |
6 | # optimization related
7 | criterion: loss
8 | early-stop-criterion: "validation/main/loss"
9 | sortagrad: 0
10 | opt: noam
11 | epochs: 15
12 | patience: 0
13 | accum-grad: 4
14 | grad-clip: 5.0
15 |
16 | # transformer related
17 | transformer-lr: 10.0
18 | transformer-warmup-steps: 8000
19 | transformer-attn-dropout-rate-encoder: 0.4
20 | transformer-attn-dropout-rate-decoder: 0.1
21 |
22 | # network architecture
23 | ## encoder related
24 | etype: transformer
25 | transformer-input-layer: conv2d
26 | elayers: 6
27 | eunits: 320
28 | dropout-rate: 0.4
29 | ## decoder related
30 | dtype: transformer
31 | dlayers: 6
32 | dec-embed-dim: 300
33 | dunits: 300
34 | dropout-rate-decoder: 0.1
35 | ## attention related
36 | adim: 320
37 | aheads: 4
38 | ## joint network related
39 | joint-dim: 300
40 |
41 | # transducer related
42 | mtlalpha: 1.0 # mtlalpha should be set to 1.0 (CTC) to use transducer
43 | rnnt-mode: 'rnnt' # switch to 'rnnt-att' to use transducer with attention
44 | model-module: espnet.nets.pytorch_backend.e2e_asr_transducer:E2E
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/tuning/decode_pytorch_transformer.yaml:
--------------------------------------------------------------------------------
1 | batchsize: 0
2 | beam-size: 5
3 | penalty: 0.0
4 | maxlenratio: 0.0
5 | minlenratio: 0.0
6 | ctc-weight: 0.0
7 | #lm-weight: 0.7
8 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/tuning/decode_rnn.yaml:
--------------------------------------------------------------------------------
1 | beam-size: 20
2 | penalty: 0.0
3 | maxlenratio: 0.0
4 | minlenratio: 0.0
5 | ctc-weight: 0.6
6 | lm-weight: 0.3
7 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/conf/tuning/train_rnn.yaml:
--------------------------------------------------------------------------------
1 | # network architecture
2 | # encoder related
3 | etype: vggblstm # encoder architecture type
4 | elayers: 3
5 | eunits: 1024
6 | eprojs: 1024
7 | subsample: "1_2_2_1_1" # skip every n frame from input to nth layers
8 | # decoder related
9 | dlayers: 2
10 | dunits: 1024
11 | # attention related
12 | atype: location
13 | adim: 1024
14 | aconv-chans: 10
15 | aconv-filts: 100
16 |
17 | # hybrid CTC/attention
18 | mtlalpha: 0.5
19 |
20 | # minibatch related
21 | batch-size: 30
22 | maxlen-in: 800 # if input length > maxlen_in, batchsize is automatically reduced
23 | maxlen-out: 150 # if output length > maxlen_out, batchsize is automatically reduced
24 |
25 | # optimization related
26 | opt: adadelta
27 | epochs: 10
28 | patience: 0
29 |
30 | # scheduled sampling option
31 | sampling-probability: 0.0
32 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/local/prepare_for_eer.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | #
3 | # Copyright 2015 David Snyder
4 | # Apache 2.0.
5 | #
6 | # Copied from egs/sre10/v1/local/prepare_for_eer.py (commit 9cb4c4c2fb0223ee90c38d98af11305074eb7ef8)
7 | #
8 | # Given a trials and scores file, this script
9 | # prepares input for the binary compute-eer.
10 | import sys
11 | trials = open(sys.argv[1], 'r').readlines()
12 | scores = open(sys.argv[2], 'r').readlines()
13 | spkrutt2target = {}
14 | for line in trials:
15 | spkr, utt, target = line.strip().split()
16 | spkrutt2target[spkr+utt]=target
17 | for line in scores:
18 | spkr, utt, score = line.strip().split()
19 | print("{} {}".format(score, spkrutt2target[spkr+utt]))
20 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/path.sh:
--------------------------------------------------------------------------------
1 | MAIN_ROOT=$PWD/../../..
2 | KALDI_ROOT=$MAIN_ROOT/tools/kaldi
3 |
4 | [ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh
5 | export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/tools/sctk/bin:$PWD:$PATH
6 | [ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1
7 | . $KALDI_ROOT/tools/config/common_path.sh
8 | export LC_ALL=C
9 |
10 | export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$MAIN_ROOT/tools/chainer_ctc/ext/warp-ctc/build
11 |
12 | #. /home/container_user/hang/espnet/tools/activate_python.sh
13 | if [ -e $MAIN_ROOT/tools/venv/etc/profile.d/conda.sh ]; then #venv
14 | source $MAIN_ROOT/tools/venv/etc/profile.d/conda.sh && conda deactivate && conda activate base #abc
15 | else
16 | source $MAIN_ROOT/tools/venv/bin/activate
17 | fi
18 |
19 | export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
20 |
21 | export OMP_NUM_THREADS=1
22 |
23 | # NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
24 | export PYTHONIOENCODING=UTF-8
25 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/chain/e2e/README.txt:
--------------------------------------------------------------------------------
1 | The scripts related to end2end chain training are in this directory
2 | Currently it has 3 scripts:
3 |
4 | ** prepare_e2e.sh which is almost equivalent
5 | to regular chain's build-tree.sh (i.e. it creates the tree and
6 | the transition-model) except it does not require any previously
7 | trained models (in other terms, it does what stages -3 and -2
8 | of steps/train_mono.sh do).
9 |
10 | ** get_egs_e2e.sh: this is simlilar to chain/get_egs.sh except it
11 | uses training FSTs (instead of lattices) to generate end2end egs.
12 |
13 | ** train_e2e.py: this is very similar to chain/train.py but
14 | with fewer stages (e.g. it does not compute the preconditioning matrix)
15 |
16 |
17 | For details please see the comments at top of local/chain/e2e/run_flatstart_*.sh
18 | and also src/chain/chain-generic-numerator.h.
19 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/data/data_dir_manipulation_lib.py:
--------------------------------------------------------------------------------
1 | import subprocess
2 |
3 | def RunKaldiCommand(command, wait = True):
4 | """ Runs commands frequently seen in Kaldi scripts. These are usually a
5 | sequence of commands connected by pipes, so we use shell=True """
6 | #logger.info("Running the command\n{0}".format(command))
7 | p = subprocess.Popen(command, shell = True,
8 | stdout = subprocess.PIPE,
9 | stderr = subprocess.PIPE)
10 |
11 | if wait:
12 | [stdout, stderr] = p.communicate()
13 | if p.returncode is not 0:
14 | raise Exception("There was an error while running the command {0}\n------------\n{1}".format(command, stderr))
15 | return stdout, stderr
16 | else:
17 | return p
18 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/libs/__init__.py:
--------------------------------------------------------------------------------
1 |
2 |
3 | # Copyright 2016 Vimal Manohar
4 | # Apache 2.0.
5 |
6 | """ This package contains modules and subpackages used in kaldi scripts.
7 | """
8 |
9 | from . import common
10 |
11 | __all__ = ["common"]
12 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/__init__.py:
--------------------------------------------------------------------------------
1 |
2 |
3 | # Copyright 2016 Johns Hopkins University (Dan Povey)
4 | # 2016 Vimal Manohar
5 | # 2016 Vijayaditya Peddinti
6 | # 2016 Yiming Wang
7 | # Apache 2.0.
8 |
9 |
10 | # This module has the python functions which facilitate the use of nnet3 toolkit
11 | # It has two sub-modules
12 | # xconfig : Library for parsing high level description of neural networks
13 | # train : Library for training scripts
14 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/report/__init__.py:
--------------------------------------------------------------------------------
1 |
2 |
3 | # Copyright 2016 Vimal Manohar
4 | # Apache 2.0.
5 |
6 | from . import log_parse
7 |
8 | __all__ = ["log_parse"]
9 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/train/__init__.py:
--------------------------------------------------------------------------------
1 |
2 | # Copyright 2016 Vimal Manohar
3 | # Apache 2.0
4 |
5 | """ This library has classes and methods commonly used for training nnet3
6 | neural networks.
7 |
8 | It has separate submodules for frame-level objectives and chain objective:
9 | frame_level_objf -- For both recurrent and non-recurrent architectures
10 | chain_objf -- LF-MMI objective training
11 | """
12 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/train/chain_objf/__init__.py:
--------------------------------------------------------------------------------
1 |
2 |
3 | # Copyright 2016 Vimal Manohar
4 | # Apache 2.0.
5 |
6 | """ This is a subpackage containing modules for training of
7 | deep neural network acoustic model with chain objective.
8 | """
9 |
10 | from . import acoustic_model
11 |
12 | __all__ = ["acoustic_model"]
13 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/train/frame_level_objf/__init__.py:
--------------------------------------------------------------------------------
1 |
2 |
3 | # Copyright 2016 Vimal Manohar
4 | # Apache 2.0
5 |
6 | """ This library has classes and methods commonly used for training nnet3
7 | neural networks with frame-level objectives.
8 | """
9 |
10 | from . import common
11 | from . import raw_model
12 | from . import acoustic_model
13 |
14 | __all__ = ["common", "raw_model", "acoustic_model"]
15 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/libs/nnet3/xconfig/layers.py:
--------------------------------------------------------------------------------
1 | # Copyright 2016 Johns Hopkins University (Dan Povey)
2 | # 2016 Vijayaditya Peddinti
3 | # 2016 Yiming Wang
4 | # Apache 2.0.
5 |
6 | from .basic_layers import *
7 | from .convolution import *
8 | from .attention import *
9 | from .lstm import *
10 | from .gru import *
11 | from .stats_layer import *
12 | from .trivial_layers import *
13 | from .composite_layers import *
14 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/nnet2/get_num_frames.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # This script works out the approximate number of frames in a training directory.
4 | # This is sometimes needed by higher-level scripts
5 |
6 |
7 | if [ -f path.sh ]; then . ./path.sh; fi
8 | . parse_options.sh || exit 1;
9 |
10 | if [ $# -ne 1 ]; then
11 | (
12 | echo "Usage: $0 "
13 | echo "Prints the number of frames of data in the data-dir"
14 | ) 1>&2
15 | fi
16 |
17 | data=$1
18 |
19 | if [ ! -f $data/utt2dur ]; then
20 | utils/data/get_utt2dur.sh $data 1>&2 || exit 1
21 | fi
22 |
23 | frame_shift=$(utils/data/get_frame_shift.sh $data) || exit 1
24 |
25 | awk -v s=$frame_shift '{n += $2} END{printf("%.0f\n", (n / s))}' <$data/utt2dur
26 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/nnet3/chain/e2e/README.txt:
--------------------------------------------------------------------------------
1 | The scripts related to end2end chain training are in this directory
2 | Currently it has 3 scripts:
3 |
4 | ** prepare_e2e.sh which is almost equivalent
5 | to regular chain's build-tree.sh (i.e. it creates the tree and
6 | the transition-model) except it does not require any previously
7 | trained models (in other terms, it does what stages -3 and -2
8 | of steps/train_mono.sh do).
9 |
10 | ** get_egs_e2e.sh: this is simlilar to chain/get_egs.sh except it
11 | uses training FSTs (instead of lattices) to generate end2end egs.
12 |
13 | ** train_e2e.py: this is very similar to chain/train.py but
14 | with fewer stages (e.g. it does not compute the preconditioning matrix)
15 |
16 |
17 | For details please see the comments at top of local/chain/e2e/run_flatstart_*.sh
18 | and also src/chain/chain-generic-numerator.h.
19 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/tfrnnlm/check_py.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import tensorflow as tf
3 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/steps/tfrnnlm/check_tensorflow_installed.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # this script checks if TF is installed to be used with python
4 | # and if TF related binaries in kaldi is ready to use
5 | . ./path.sh
6 |
7 | if which lattice-lmrescore-tf-rnnlm 2>&1>/dev/null; then
8 | echo TensorFlow relate binaries found. This is good.
9 | else
10 | echo TF related binaries not compiled.
11 | echo You need to go to tools/ and run extras/install_tensorflow_cc.sh first
12 | echo and then do \"make\" under both src/tfrnnlm and src/tfrnnlmbin
13 | exit 1
14 | fi
15 |
16 | echo
17 |
18 | if python steps/tfrnnlm/check_py.py 2>/dev/null; then
19 | echo TensorFlow ready to use on the python side. This is good.
20 | else
21 | echo TensorFlow not found on the python side.
22 | echo Please go to tools/ and run extras/install_tensorflow_py.sh to install it
23 | echo If you already have TensorFlow installed somewhere else, you would need
24 | echo to add it to your PATH
25 | exit 1
26 | fi
27 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/ctm/fix_ctm.sh:
--------------------------------------------------------------------------------
1 | #! /bin/bash
2 |
3 | stmfile=$1
4 | ctmfile=$2
5 |
6 | segments_stm=`cat $stmfile | cut -f 1 -d ' ' | sort -u`
7 | segments_ctm=`cat $ctmfile | cut -f 1 -d ' ' | sort -u`
8 |
9 | segments_stm_count=`echo "$segments_stm" | wc -l `
10 | segments_ctm_count=`echo "$segments_ctm" | wc -l `
11 |
12 | #echo $segments_stm_count
13 | #echo $segments_ctm_count
14 |
15 | if [ "$segments_stm_count" -gt "$segments_ctm_count" ] ; then
16 | pp=$( diff <(echo "$segments_stm") <(echo "$segments_ctm" ) | grep "^<" | sed "s/^< *//g")
17 | (
18 | for elem in $pp ; do
19 | echo "$elem 1 0 0 EMPTY_RECOGNIZED_PHRASE"
20 | done
21 | ) >> $ctmfile
22 | echo "FIXED CTM FILE"
23 | exit 0
24 | elif [ "$segments_stm_count" -lt "$segments_ctm_count" ] ; then
25 | echo "Segment STM count: $segments_stm_count"
26 | echo "Segment CTM count: $segments_ctm_count"
27 | echo "FAILURE FIXING CTM FILE"
28 | exit 1
29 | else
30 | exit 0
31 | fi
32 |
33 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/data/get_num_frames.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # This script works out the approximate number of frames in a training directory.
4 | # This is sometimes needed by higher-level scripts
5 |
6 |
7 | if [ -f path.sh ]; then . ./path.sh; fi
8 | . parse_options.sh || exit 1;
9 |
10 | if [ $# -ne 1 ]; then
11 | (
12 | echo "Usage: $0 "
13 | echo "Prints the number of frames of data in the data-dir"
14 | ) 1>&2
15 | fi
16 |
17 | data=$1
18 |
19 | if [ ! -f $data/utt2dur ]; then
20 | utils/data/get_utt2dur.sh $data 1>&2 || exit 1
21 | fi
22 |
23 | frame_shift=$(utils/data/get_frame_shift.sh $data) || exit 1
24 |
25 | awk -v s=$frame_shift '{n += $2} END{printf("%.0f\n", (n / s))}' <$data/utt2dur
26 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/data/get_reco2utt_for_data.sh:
--------------------------------------------------------------------------------
1 | #! /bin/bash
2 |
3 | # Copyright 2016 Vimal Manohar
4 | # Apache 2.0
5 |
6 | if [ $# -ne 1 ]; then
7 | echo "This script outputs a mapping from recording to a list of utterances "
8 | echo "corresponding to the recording. It is analogous to the content of "
9 | echo "a spk2utt file, but is indexed by recording instead of speaker."
10 | echo "Usage: get_reco2utt.sh "
11 | echo " e.g.: get_reco2utt.sh data/train"
12 | exit 1
13 | fi
14 |
15 | data=$1
16 |
17 | if [ ! -s $data/segments ]; then
18 | utils/data/get_segments_for_data.sh $data > $data/segments
19 | fi
20 |
21 | cut -d ' ' -f 1,2 $data/segments | utils/utt2spk_to_spk2utt.pl
22 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/data/get_segments_for_data.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # This script operates on a data directory, such as in data/train/,
4 | # and writes new segments to stdout. The file 'segments' maps from
5 | # utterance to time offsets into a recording, with the format:
6 | #
7 | # This script assumes utterance and recording ids are the same (i.e., that
8 | # wav.scp is indexed by utterance), and uses durations from 'utt2dur',
9 | # created if necessary by get_utt2dur.sh.
10 |
11 | . ./path.sh
12 |
13 | if [ $# != 1 ]; then
14 | echo "Usage: $0 [options] "
15 | echo "e.g.:"
16 | echo " $0 data/train > data/train/segments"
17 | exit 1
18 | fi
19 |
20 | data=$1
21 |
22 | if [ ! -s $data/utt2dur ]; then
23 | utils/data/get_utt2dur.sh $data 1>&2 || exit 1;
24 | fi
25 |
26 | # 0
27 | awk '{ print $1, $1, 0, $2 }' $data/utt2dur
28 |
29 | exit 0
30 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/eps2disambig.pl:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env perl
2 | # Copyright 2010-2011 Microsoft Corporation
3 | # 2015 Guoguo Chen
4 |
5 | # Licensed under the Apache License, Version 2.0 (the "License");
6 | # you may not use this file except in compliance with the License.
7 | # You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
12 | # KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
13 | # WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
14 | # MERCHANTABLITY OR NON-INFRINGEMENT.
15 | # See the Apache 2 License for the specific language governing permissions and
16 | # limitations under the License.
17 |
18 | # This script replaces epsilon with #0 on the input side only, of the G.fst
19 | # acceptor.
20 |
21 | while(<>){
22 | if (/\s+#0\s+/) {
23 | print STDERR "$0: ERROR: LM has word #0, " .
24 | "which is reserved as disambiguation symbol\n";
25 | exit 1;
26 | }
27 | s:^(\d+\s+\d+\s+)\(\s+):$1#0$2:;
28 | print;
29 | }
30 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/filt.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | # Apache 2.0
4 |
5 | from __future__ import print_function
6 | import sys
7 |
8 | vocab=set()
9 | with open(sys.argv[1]) as vocabfile:
10 | for line in vocabfile:
11 | vocab.add(line.strip())
12 |
13 | with open(sys.argv[2]) as textfile:
14 | for line in textfile:
15 | print(" ".join([word if word in vocab else '' for word in line.strip().split()]))
16 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/fix_ctm.sh:
--------------------------------------------------------------------------------
1 | #! /bin/bash
2 |
3 | stmfile=$1
4 | ctmfile=$2
5 |
6 | segments_stm=`cat $stmfile | cut -f 1 -d ' ' | sort -u`
7 | segments_ctm=`cat $ctmfile | cut -f 1 -d ' ' | sort -u`
8 |
9 | segments_stm_count=`echo "$segments_stm" | wc -l `
10 | segments_ctm_count=`echo "$segments_ctm" | wc -l `
11 |
12 | #echo $segments_stm_count
13 | #echo $segments_ctm_count
14 |
15 | if [ "$segments_stm_count" -gt "$segments_ctm_count" ] ; then
16 | pp=$( diff <(echo "$segments_stm") <(echo "$segments_ctm" ) | grep "^<" | sed "s/^< *//g")
17 | (
18 | for elem in $pp ; do
19 | echo "$elem 1 0 0 EMPTY_RECOGNIZED_PHRASE"
20 | done
21 | ) >> $ctmfile
22 | echo "FIXED CTM FILE"
23 | exit 0
24 | elif [ "$segments_stm_count" -lt "$segments_ctm_count" ] ; then
25 | echo "Segment STM count: $segments_stm_count"
26 | echo "Segment CTM count: $segments_ctm_count"
27 | echo "FAILURE FIXING CTM FILE"
28 | exit 1
29 | else
30 | exit 0
31 | fi
32 |
33 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/lang/bpe/prepend_words.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | # This script, prepend '|' to every words in the transcript to mark
4 | # the beginning of the words for finding the initial-space of every word
5 | # after decoding.
6 |
7 | import sys
8 | import io
9 | import re
10 |
11 | whitespace = re.compile("[ \t]+")
12 | infile = io.TextIOWrapper(sys.stdin.buffer, encoding='latin-1')
13 | output = io.TextIOWrapper(sys.stdout.buffer, encoding='latin-1')
14 | for line in infile:
15 | words = whitespace.split(line.strip(" \t\r\n"))
16 | output.write(' '.join([ "|"+word for word in words]) + '\n')
17 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/lang/bpe/reverse.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 |
4 | # This script, reverse all latin and digits sequences
5 | # (including words like MP3) to put them in the right order in the images.
6 |
7 | import re, os, sys, io
8 |
9 | in_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
10 | out_stream = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
11 | for line in in_stream:
12 | out_stream.write(re.sub(r'[a-zA-Z0-9][a-zA-Z0-9\s\.\:]*[a-zA-Z0-9]',
13 | lambda m:m.group(0)[::-1], line))
14 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/make_absolute.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # This script replaces the command readlink -f (which is not portable).
4 | # It turns a pathname into an absolute pathname, including following soft links.
5 | target_file=$1
6 |
7 | cd $(dirname $target_file)
8 | target_file=$(basename $target_file)
9 |
10 | # Iterate down a (possible) chain of symlinks
11 | while [ -L "$target_file" ]; do
12 | target_file=$(readlink $target_file)
13 | cd $(dirname $target_file)
14 | target_file=$(basename $target_file)
15 | done
16 |
17 | # Compute the canonicalized name by finding the physical path
18 | # for the directory we're in and appending the target file.
19 | phys_dir=$(pwd -P)
20 | result=$phys_dir/$target_file
21 | echo $result
22 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/s2eps.pl:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env perl
2 | # Copyright 2010-2011 Microsoft Corporation
3 |
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
11 | # KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
12 | # WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
13 | # MERCHANTABLITY OR NON-INFRINGEMENT.
14 | # See the Apache 2 License for the specific language governing permissions and
15 | # limitations under the License.
16 |
17 | # This script replaces and with (on both input and output sides),
18 | # for the G.fst acceptor.
19 |
20 | while(<>){
21 | @A = split(" ", $_);
22 | if ( @A >= 4 ) {
23 | if ($A[2] eq "" || $A[2] eq "") { $A[2] = ""; }
24 | if ($A[3] eq "" || $A[3] eq "") { $A[3] = ""; }
25 | }
26 | print join("\t", @A) . "\n";
27 | }
28 |
--------------------------------------------------------------------------------
/Speech_Recognition/egs/aishell/asr/utils/spk2utt_to_utt2spk.pl:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env perl
2 | # Copyright 2010-2011 Microsoft Corporation
3 |
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
11 | # KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
12 | # WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
13 | # MERCHANTABLITY OR NON-INFRINGEMENT.
14 | # See the Apache 2 License for the specific language governing permissions and
15 | # limitations under the License.
16 |
17 |
18 | while(<>){
19 | @A = split(" ", $_);
20 | @A > 1 || die "Invalid line in spk2utt file: $_";
21 | $s = shift @A;
22 | foreach $u ( @A ) {
23 | print "$u $s\n";
24 | }
25 | }
26 |
27 |
28 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize main package."""
2 |
3 | import pkg_resources
4 |
5 | try:
6 | __version__ = pkg_resources.get_distribution("espnet").version
7 | except Exception:
8 | __version__ = "(Not installed from setup.py)"
9 | del pkg_resources
10 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/asr/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/asr/chainer_backend/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/asr/pytorch_backend/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/bin/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/lm/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/lm/chainer_backend/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/lm/pytorch_backend/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/mt/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/mt/pytorch_backend/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/chainer_backend/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/chainer_backend/asr_interface.py:
--------------------------------------------------------------------------------
1 | """ASR Interface module."""
2 | import chainer
3 |
4 | from espnet.nets.asr_interface import ASRInterface
5 |
6 |
7 | class ChainerASRInterface(ASRInterface, chainer.Chain):
8 | """ASR Interface for ESPnet model implementation."""
9 |
10 | @staticmethod
11 | def custom_converter(*args, **kw):
12 | """Get customconverter of the model (Chainer only)."""
13 | raise NotImplementedError("custom converter method is not implemented")
14 |
15 | @staticmethod
16 | def custom_updater(*args, **kw):
17 | """Get custom_updater of the model (Chainer only)."""
18 | raise NotImplementedError("custom updater method is not implemented")
19 |
20 | @staticmethod
21 | def custom_parallel_updater(*args, **kw):
22 | """Get custom_parallel_updater of the model (Chainer only)."""
23 | raise NotImplementedError("custom parallel updater method is not implemented")
24 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/chainer_backend/nets_utils.py:
--------------------------------------------------------------------------------
1 | import chainer.functions as F
2 |
3 |
4 | def _subsamplex(x, n):
5 | x = [F.get_item(xx, (slice(None, None, n), slice(None))) for xx in x]
6 | ilens = [xx.shape[0] for xx in x]
7 | return x, ilens
8 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/chainer_backend/rnn/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/chainer_backend/transformer/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/chainer_backend/transformer/layer_norm.py:
--------------------------------------------------------------------------------
1 | # encoding: utf-8
2 | """Class Declaration of Transformer's Label Smootion loss."""
3 |
4 | import chainer.links as L
5 |
6 |
7 | class LayerNorm(L.LayerNormalization):
8 | """Redirect to L.LayerNormalization."""
9 |
10 | def __init__(self, dims, eps=1e-12):
11 | """Initialize LayerNorm."""
12 | super(LayerNorm, self).__init__(size=dims, eps=eps)
13 |
14 | def __call__(self, e):
15 | """Forward LayerNorm."""
16 | return super(LayerNorm, self).__call__(e)
17 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/chainer_backend/transformer/mask.py:
--------------------------------------------------------------------------------
1 | """Create mask for subsequent steps."""
2 |
3 |
4 | def make_history_mask(xp, block):
5 | """Prepare the history mask.
6 |
7 | Args:
8 | block (ndarray): Block with dimensions: (B x S).
9 | Returns:
10 | ndarray, np.ndarray: History mask with dimensions (B, S, S).
11 |
12 | """
13 | batch, length = block.shape
14 | arange = xp.arange(length)
15 | history_mask = (arange[None] <= arange[:, None])[
16 | None,
17 | ]
18 | history_mask = xp.broadcast_to(history_mask, (batch, length, length))
19 | return history_mask
20 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/compressive_transducer/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/fastspeech/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/frontends/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/lm/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/rnn/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/streaming/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/sync_transducer/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/tacotron2/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/transducer/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/transformer/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/transformer/add_sos_eos.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 |
4 | # Copyright 2019 Shigeki Karita
5 | # Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
6 |
7 | """Unility funcitons for Transformer."""
8 |
9 | import torch
10 |
11 |
12 | def add_sos_eos(ys_pad, sos, eos, ignore_id):
13 | """Add and labels.
14 |
15 | :param torch.Tensor ys_pad: batch of padded target sequences (B, Lmax)
16 | :param int sos: index of
17 | :param int eos: index of
18 | :param int ignore_id: index of padding
19 | :return: padded tensor (B, Lmax)
20 | :rtype: torch.Tensor
21 | :return: padded tensor (B, Lmax)
22 | :rtype: torch.Tensor
23 | """
24 | from espnet.nets.pytorch_backend.nets_utils import pad_list
25 |
26 | _sos = ys_pad.new([sos])
27 | _eos = ys_pad.new([eos])
28 | ys = [y[y != ignore_id] for y in ys_pad] # parse padded ys
29 | ys_in = [torch.cat([_sos, y], dim=0) for y in ys]
30 | ys_out = [torch.cat([y, _eos], dim=0) for y in ys]
31 | return pad_list(ys_in, eos), pad_list(ys_out, ignore_id)
32 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/transformer/layer_norm.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 |
4 | # Copyright 2019 Shigeki Karita
5 | # Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
6 |
7 | """Layer normalization module."""
8 |
9 | import torch
10 |
11 |
12 | class LayerNorm(torch.nn.LayerNorm):
13 | """Layer normalization module.
14 |
15 | :param int nout: output dim size
16 | :param int dim: dimension to be normalized
17 | """
18 |
19 | def __init__(self, nout, dim=-1):
20 | """Construct an LayerNorm object."""
21 | super(LayerNorm, self).__init__(nout, eps=1e-12)
22 | self.dim = dim
23 |
24 | def forward(self, x):
25 | """Apply layer normalization.
26 |
27 | :param torch.Tensor x: input tensor
28 | :return: layer normalized tensor
29 | :rtype torch.Tensor
30 | """
31 | if self.dim == -1:
32 | return super(LayerNorm, self).forward(x)
33 | return super(LayerNorm, self).forward(x.transpose(1, -1)).transpose(1, -1)
34 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/pytorch_backend/transformer/repeat.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 |
4 | # Copyright 2019 Shigeki Karita
5 | # Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
6 |
7 | """Repeat the same layer definition."""
8 |
9 | import torch
10 |
11 |
12 | class MultiSequential(torch.nn.Sequential):
13 | """Multi-input multi-output torch.nn.Sequential."""
14 |
15 | def forward(self, *args):
16 | """Repeat."""
17 | for m in self:
18 | args = m(*args)
19 | return args
20 |
21 |
22 | def repeat(N, fn):
23 | """Repeat module N times.
24 |
25 | :param int N: repeat time
26 | :param function fn: function to generate module
27 | :return: repeated modules
28 | :rtype: MultiSequential
29 | """
30 | return MultiSequential(*[fn() for _ in range(N)])
31 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/nets/scorers/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/optimizer/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/optimizer/parser.py:
--------------------------------------------------------------------------------
1 | """Common optimizer default config for multiple backends."""
2 |
3 |
4 | def sgd(parser):
5 | """Add arguments."""
6 | parser.add_argument("--lr", type=float, default=1.0, help="Learning rate")
7 | parser.add_argument("--weight-decay", type=float, default=0.0, help="Weight decay")
8 | return parser
9 |
10 |
11 | def adam(parser):
12 | """Add arguments."""
13 | parser.add_argument("--lr", type=float, default=1e-3, help="Learning rate")
14 | parser.add_argument("--beta1", type=float, default=0.9, help="Beta1")
15 | parser.add_argument("--beta2", type=float, default=0.999, help="Beta2")
16 | parser.add_argument("--weight-decay", type=float, default=0.0, help="Weight decay")
17 | return parser
18 |
19 |
20 | def adadelta(parser):
21 | """Add arguments."""
22 | parser.add_argument("--rho", type=float, default=0.95, help="Rho")
23 | parser.add_argument("--eps", type=float, default=1e-8, help="Eps")
24 | parser.add_argument("--weight-decay", type=float, default=0.0, help="Weight decay")
25 | return parser
26 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/scheduler/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/scheduler/chainer.py:
--------------------------------------------------------------------------------
1 | """Chainer optimizer schdulers."""
2 |
3 | from typing import List
4 |
5 | from chainer.optimizer import Optimizer
6 |
7 | from espnet.scheduler.scheduler import SchedulerInterface
8 |
9 |
10 | class ChainerScheduler:
11 | """Chainer optimizer scheduler."""
12 |
13 | def __init__(self, schedulers: List[SchedulerInterface], optimizer: Optimizer):
14 | """Initialize class."""
15 | self.schedulers = schedulers
16 | self.optimizer = optimizer
17 | self.init_values = dict()
18 | for s in self.schedulers:
19 | self.init_values[s.key] = getattr(self.optimizer, s.key)
20 |
21 | def step(self, n_iter: int):
22 | """Update optimizer by scheduling."""
23 | for s in self.schedulers:
24 | new_val = self.init_values[s.key] * s.scale(n_iter)
25 | setattr(self.optimizer, s.key, new_val)
26 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/scheduler/pytorch.py:
--------------------------------------------------------------------------------
1 | """PyTorch optimizer schdulers."""
2 |
3 | from typing import List
4 |
5 | from torch.optim import Optimizer
6 |
7 | from espnet.scheduler.scheduler import SchedulerInterface
8 |
9 |
10 | class PyTorchScheduler:
11 | """PyTorch optimizer scheduler."""
12 |
13 | def __init__(self, schedulers: List[SchedulerInterface], optimizer: Optimizer):
14 | """Initialize class."""
15 | self.schedulers = schedulers
16 | self.optimizer = optimizer
17 | for s in self.schedulers:
18 | for group in optimizer.param_groups:
19 | group.setdefault("initial_" + s.key, group[s.key])
20 |
21 | def step(self, n_iter: int):
22 | """Update optimizer by scheduling."""
23 | for s in self.schedulers:
24 | for group in self.optimizer.param_groups:
25 | group[s.key] = group["initial_" + s.key] * s.scale(n_iter)
26 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/st/pytorch_backend/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/transform/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize main package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/transform/add_deltas.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | def delta(feat, window):
5 | assert window > 0
6 | delta_feat = np.zeros_like(feat)
7 | for i in range(1, window + 1):
8 | delta_feat[:-i] += i * feat[i:]
9 | delta_feat[i:] += -i * feat[:-i]
10 | delta_feat[-i:] += i * feat[-1]
11 | delta_feat[:i] += -i * feat[0]
12 | delta_feat /= 2 * sum(i ** 2 for i in range(1, window + 1))
13 | return delta_feat
14 |
15 |
16 | def add_deltas(x, window=2, order=2):
17 | feats = [x]
18 | for _ in range(order):
19 | feats.append(delta(feats[-1], window))
20 | return np.concatenate(feats, axis=1)
21 |
22 |
23 | class AddDeltas(object):
24 | def __init__(self, window=2, order=2):
25 | self.window = window
26 | self.order = order
27 |
28 | def __repr__(self):
29 | return "{name}(window={window}, order={order}".format(
30 | name=self.__class__.__name__, window=self.window, order=self.order
31 | )
32 |
33 | def __call__(self, x):
34 | return add_deltas(x, window=self.window, order=self.order)
35 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/transform/transform_interface.py:
--------------------------------------------------------------------------------
1 | # TODO(karita): add this to all the transform impl.
2 | class TransformInterface:
3 | """Transform Interface"""
4 |
5 | def __call__(self, x):
6 | raise NotImplementedError("__call__ method is not implemented")
7 |
8 | @classmethod
9 | def add_arguments(cls, parser):
10 | return parser
11 |
12 | def __repr__(self):
13 | return self.__class__.__name__ + "()"
14 |
15 |
16 | class Identity(TransformInterface):
17 | """Identity Function"""
18 |
19 | def __call__(self, x):
20 | return x
21 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/tts/pytorch_backend/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/utils/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/utils/check_kwargs.py:
--------------------------------------------------------------------------------
1 | import inspect
2 |
3 |
4 | def check_kwargs(func, kwargs, name=None):
5 | """check kwargs are valid for func
6 |
7 | If kwargs are invalid, raise TypeError as same as python default
8 | :param function func: function to be validated
9 | :param dict kwargs: keyword arguments for func
10 | :param str name: name used in TypeError (default is func name)
11 | """
12 | try:
13 | params = inspect.signature(func).parameters
14 | except ValueError:
15 | return
16 | if name is None:
17 | name = func.__name__
18 | for k in kwargs.keys():
19 | if k not in params:
20 | raise TypeError(f"{name}() got an unexpected keyword argument '{k}'")
21 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/utils/dynamic_import.py:
--------------------------------------------------------------------------------
1 | import importlib
2 |
3 |
4 | def dynamic_import(import_path, alias=dict()):
5 | """dynamic import module and class
6 |
7 | :param str import_path: syntax 'module_name:class_name'
8 | e.g., 'espnet.transform.add_deltas:AddDeltas'
9 | :param dict alias: shortcut for registered class
10 | :return: imported class
11 | """
12 | if import_path not in alias and ":" not in import_path:
13 | raise ValueError(
14 | "import_path should be one of {} or "
15 | 'include ":", e.g. "espnet.transform.add_deltas:AddDeltas" : '
16 | "{}".format(set(alias), import_path)
17 | )
18 | if ":" not in import_path:
19 | import_path = alias[import_path]
20 |
21 | module_name, objname = import_path.split(":")
22 | m = importlib.import_module(module_name)
23 | return getattr(m, objname)
24 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/utils/training/__init__.py:
--------------------------------------------------------------------------------
1 | """Initialize sub package."""
2 |
--------------------------------------------------------------------------------
/Speech_Recognition/espnet/utils/training/evaluator.py:
--------------------------------------------------------------------------------
1 | from chainer.training.extensions import Evaluator
2 |
3 | from espnet.utils.training.tensorboard_logger import TensorboardLogger
4 |
5 |
6 | class BaseEvaluator(Evaluator):
7 | """Base Evaluator in ESPnet"""
8 |
9 | def __call__(self, trainer=None):
10 | ret = super().__call__(trainer)
11 | try:
12 | if trainer is not None:
13 | # force tensorboard to report evaluation log
14 | tb_logger = trainer.get_extension(TensorboardLogger.default_name)
15 | tb_logger(trainer)
16 | except ValueError:
17 | pass
18 | return ret
19 |
--------------------------------------------------------------------------------
/Speech_Recognition/setup.cfg:
--------------------------------------------------------------------------------
1 | [aliases]
2 | test=pytest
3 |
4 | [tool:pytest]
5 | addopts = --verbose --durations=0 --cov=espnet
6 | testpaths = test
7 |
8 | [flake8]
9 | ignore = H102,W504,H238
10 | # 120 is a workaround, 79 is good
11 | max-line-length = 120
12 |
--------------------------------------------------------------------------------
/Speech_Recognition/tools/installers/install_chainer_ctc.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 | if [ $# != 0 ]; then
5 | echo "Usage: $0"
6 | exit 1;
7 | fi
8 |
9 | rm -rf chainer_ctc
10 | git clone https://github.com/jheymann85/chainer_ctc.git
11 | python3 -m pip install cython
12 | (
13 | set -euo pipefail
14 | cd chainer_ctc && chmod +x install_warp-ctc.sh && ./install_warp-ctc.sh && python3 -m pip install .
15 | )
16 |
--------------------------------------------------------------------------------
/Speech_Recognition/tools/installers/install_kenlm.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 | if [ $# != 0 ]; then
5 | echo "Usage: $0"
6 | exit 1;
7 | fi
8 |
9 | rm -rf kenlm
10 | git clone https://github.com/kpu/kenlm.git
11 | (
12 | set -euo pipefail
13 | cd kenlm
14 |
15 | mkdir build
16 | (
17 | set -euo pipefail
18 | cd build && cmake .. && make
19 | )
20 | (
21 | set -euo pipefail
22 | python3 setup.py install
23 | )
24 | )
25 |
--------------------------------------------------------------------------------
/Speech_Recognition/tools/installers/install_mwerSegmenter.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 |
5 | if [ $# != 0 ]; then
6 | echo "Usage: $0"
7 | exit 1;
8 | fi
9 |
10 | wget --tries=3 https://www-i6.informatik.rwth-aachen.de/web/Software/mwerSegmenter.tar.gz
11 | tar zxvf mwerSegmenter.tar.gz
12 | rm mwerSegmenter.tar.gz
13 |
--------------------------------------------------------------------------------
/Speech_Recognition/tools/installers/install_nkf.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 |
5 | if [ $# != 0 ]; then
6 | echo "Usage: $0"
7 | exit 1;
8 | fi
9 |
10 |
11 | rm -rf nkf
12 | mkdir -p nkf
13 | (
14 | set -euo pipefail
15 | cd nkf
16 | wget --tries=3 https://ja.osdn.net/dl/nkf/nkf-2.1.4.tar.gz
17 | tar zxvf nkf-2.1.4.tar.gz
18 | (
19 | set -euo pipefail
20 | cd nkf-2.1.4
21 | make prefix=.
22 | )
23 | )
24 |
--------------------------------------------------------------------------------
/Speech_Recognition/tools/installers/install_pesq.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 |
5 | if [ $# != 0 ]; then
6 | echo "Usage: $0"
7 | exit 1;
8 | fi
9 |
10 | if [ ! -e PESQ.zip ]; then
11 | wget --tries=3 'http://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-P.862-200511-I!Amd2!SOFT-ZST-E&type=items' -O PESQ.zip
12 | fi
13 | if [ ! -e PESQ ]; then
14 | mkdir -p PESQ_P.862.2
15 | unzip PESQ.zip -d PESQ_P.862.2
16 | unzip "PESQ_P.862.2/Software/P862_annex_A_2005_CD wav final.zip" -d PESQ_P.862.2
17 | rm -rf PESQ
18 | ln -s PESQ_P.862.2 PESQ
19 | fi
20 |
21 | (
22 | set -euo pipefail
23 | cd PESQ/P862_annex_A_2005_CD/source
24 | gcc ./*.c -lm -o PESQ
25 | )
26 |
--------------------------------------------------------------------------------
/Speech_Recognition/tools/installers/install_sctk.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 |
5 | if [ $# != 0 ]; then
6 | echo "Usage: $0"
7 | exit 1;
8 | fi
9 |
10 |
11 | if [ ! -e sctk-2.4.10-20151007-1312Z.tar.bz2 ]; then
12 | wget -T 10 https://github.com/espnet/kaldi-bin/releases/download/v0.0.2/sctk-2.4.10-20151007-1312Z.tar.bz2 || \
13 | wget -T 10 -t 3 ftp://jaguar.ncsl.nist.gov/pub/sctk-2.4.10-20151007-1312Z.tar.bz2|| \
14 | wget --no-check-certificate -T 10 http://www.openslr.org/resources/4/sctk-2.4.10-20151007-1312Z.tar.bz2
15 | fi
16 |
17 | if [ ! -e sctk ]; then
18 | tar xojf sctk-2.4.10-20151007-1312Z.tar.bz2 || \
19 | tar --exclude '*NONE*html' -xvojf sctk-2.4.10-20151007-1312Z.tar.bz2
20 | rm -rf sctk && ln -s sctk-2.4.10 sctk
21 | fi
22 |
23 | (
24 | set -euo pipefail
25 | cd sctk
26 | make config
27 | touch .configured
28 | make all && make install && make doc
29 | )
30 |
--------------------------------------------------------------------------------
/Speech_Recognition/tools/installers/install_sph2pipe.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 |
5 | if [ $# != 0 ]; then
6 | echo "Usage: $0"
7 | exit 1;
8 | fi
9 |
10 |
11 | if [ ! -e sph2pipe_v2.5.tar.gz ]; then
12 | wget -T 10 https://github.com/espnet/kaldi-bin/releases/download/v0.0.2/sph2pipe_v2.5.tar.gz || \
13 | wget -T 10 -t 3 http://www.openslr.org/resources/3/sph2pipe_v2.5.tar.gz || \
14 | wget --no-check-certificate -T 10 https://sourceforge.net/projects/kaldi/files/sph2pipe_v2.5.tar.gz
15 | fi
16 |
17 | if [ ! -e sph2pipe_v2.5 ]; then
18 | tar --no-same-owner -xzf sph2pipe_v2.5.tar.gz
19 | fi
20 |
21 | (
22 | set -euo pipefail
23 | cd sph2pipe_v2.5
24 | gcc -o sph2pipe *.c -lm
25 | )
26 |
--------------------------------------------------------------------------------
/Speech_Recognition/tools/installers/install_warp-transducer.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 |
5 | if [ $# != 0 ]; then
6 | echo "Usage: $0"
7 | exit 1;
8 | fi
9 |
10 | # TODO(kamo): Consider clang case
11 | # Note: Requires gcc>=4.9.2 to build extensions with pytorch>=1.0
12 | if python3 -c 'import torch as t;assert t.__version__[0] == "1"' &> /dev/null; then \
13 | python3 -c "from distutils.version import LooseVersion as V;assert V('$(gcc -dumpversion)') >= V('4.9.2'), 'Requires gcc>=4.9.2'"; \
14 | fi
15 |
16 | rm -rf warp-transducer
17 | git clone https://github.com/HawkAaron/warp-transducer.git
18 |
19 | (
20 | set -euo pipefail
21 | cd warp-transducer
22 |
23 | mkdir build
24 | (
25 | set -euo pipefail
26 | cd build && cmake .. && make
27 | )
28 |
29 | (
30 | set -euo pipefail
31 | cd pytorch_binding && python3 setup.py install
32 | )
33 | )
34 |
--------------------------------------------------------------------------------
/Speech_Recognition/utils/feats2npy.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # coding: utf-8
3 |
4 | import argparse
5 | from kaldiio import ReadHelper
6 | import numpy as np
7 | import os
8 | from os.path import join
9 | import sys
10 |
11 |
12 | def get_parser():
13 | parser = argparse.ArgumentParser(
14 | description='Convet kaldi-style features to numpy arrays',
15 | formatter_class=argparse.ArgumentDefaultsHelpFormatter)
16 | parser.add_argument('scp_file', type=str, help='scp file')
17 | parser.add_argument('out_dir', type=str, help='output directory')
18 | return parser
19 |
20 |
21 | if __name__ == "__main__":
22 | args = get_parser().parse_args(sys.argv[1:])
23 | os.makedirs(args.out_dir, exist_ok=True)
24 | with ReadHelper(f"scp:{args.scp_file}") as f:
25 | for utt_id, arr in f:
26 | out_path = join(args.out_dir, f"{utt_id}-feats.npy")
27 | np.save(out_path, arr, allow_pickle=False)
28 | sys.exit(0)
29 |
--------------------------------------------------------------------------------
/Speech_Recognition/utils/get_yaml.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | import argparse
3 |
4 | import yaml
5 |
6 |
7 | def get_parser():
8 | parser = argparse.ArgumentParser(
9 | description='get a specified attribute from a YAML file',
10 | formatter_class=argparse.ArgumentDefaultsHelpFormatter)
11 | parser.add_argument('inyaml')
12 | parser.add_argument('attr', help='foo.bar will access yaml.load(inyaml)["foo"]["bar"]')
13 | return parser
14 |
15 |
16 | def main():
17 | args = get_parser().parse_args()
18 | with open(args.inyaml, 'r') as f:
19 | indict = yaml.load(f, Loader=yaml.Loader)
20 |
21 | try:
22 | for attr in args.attr.split('.'):
23 | if attr.isdigit():
24 | attr = int(attr)
25 | indict = indict[attr]
26 | print(indict)
27 | except KeyError:
28 | # print nothing
29 | # sys.exit(1)
30 | pass
31 |
32 |
33 | if __name__ == '__main__':
34 | main()
35 |
--------------------------------------------------------------------------------
/Speech_Recognition/utils/spm_train:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # Copyright (c) Facebook, Inc. and its affiliates.
3 | # All rights reserved.
4 | #
5 | # This source code is licensed under the license found in the
6 | # https://github.com/pytorch/fairseq/blob/master/LICENSE
7 | import sys
8 |
9 | import sentencepiece as spm
10 |
11 |
12 | if __name__ == "__main__":
13 | spm.SentencePieceTrainer.Train(" ".join(sys.argv[1:]))
14 |
--------------------------------------------------------------------------------