├── .gitignore ├── .gitlab-ci.yml ├── .travis.yml ├── .travis └── install.sh ├── BENCHMARK.md ├── LICENSE ├── README-NJUNMT.md ├── README-zh.md ├── README.md ├── changelog.md ├── configs ├── dl4mt_nist_zh2en.yaml ├── ed │ ├── iwslt_de2en_mgnmt.yaml │ ├── iwslt_de2en_mgnmt_np.yaml │ ├── iwslt_de2en_mgnmt_np_partial.yaml │ └── iwslt_de2en_transformer.yaml ├── test_mgnmt.yaml ├── test_mgnmt_np.yaml ├── transformer_nist_zh2en.yaml ├── transformer_nist_zh2en_bpe.yaml ├── transformer_wmt14_en2de.yaml ├── transformer_wmt15_en2fr.yaml └── transformer_wmt16_en2ro.yaml ├── requirements.txt ├── scripts ├── build_dictionary.py ├── checkpoints_average.py ├── postprocess.sh ├── train.sh └── translate.sh ├── src ├── __init__.py ├── bin │ ├── __init__.py │ ├── ensemble_translate.py │ ├── train.py │ ├── translate.py │ └── translate_mgnmt.py ├── data │ ├── __init__.py │ ├── bpe.py │ ├── data_iterator.py │ ├── dataset.py │ ├── tokenizer.py │ └── vocabulary.py ├── decoder │ ├── __init__.py │ ├── rnn_decoder.py │ └── transformer_decoder.py ├── decoding │ ├── __init__.py │ ├── beam_search.py │ ├── ensemble_beam_search.py │ ├── iterative_decoding.py │ └── utils.py ├── encoder │ ├── __init__.py │ ├── rnn_encoder.py │ └── transformer_encoder.py ├── metric │ ├── __init__.py │ ├── bleu_scorer.py │ └── scripts │ │ ├── generic │ │ ├── multi-bleu-detok.perl │ │ └── multi-bleu.perl │ │ ├── recaser │ │ └── detruecase.perl │ │ ├── share │ │ └── nonbreaking_prefixes │ │ │ ├── README.txt │ │ │ ├── nonbreaking_prefix.ca │ │ │ ├── nonbreaking_prefix.cs │ │ │ ├── nonbreaking_prefix.de │ │ │ ├── nonbreaking_prefix.el │ │ │ ├── nonbreaking_prefix.en │ │ │ ├── nonbreaking_prefix.es │ │ │ ├── nonbreaking_prefix.fi │ │ │ ├── nonbreaking_prefix.fr │ │ │ ├── nonbreaking_prefix.ga │ │ │ ├── nonbreaking_prefix.hu │ │ │ ├── nonbreaking_prefix.is │ │ │ ├── nonbreaking_prefix.it │ │ │ ├── nonbreaking_prefix.lt │ │ │ ├── nonbreaking_prefix.lv │ │ │ ├── nonbreaking_prefix.nl │ │ │ ├── nonbreaking_prefix.pl │ │ │ ├── nonbreaking_prefix.pt │ │ │ ├── nonbreaking_prefix.ro │ │ │ ├── nonbreaking_prefix.ru │ │ │ ├── nonbreaking_prefix.sk │ │ │ ├── nonbreaking_prefix.sl │ │ │ ├── nonbreaking_prefix.sv │ │ │ ├── nonbreaking_prefix.ta │ │ │ ├── nonbreaking_prefix.yue │ │ │ └── nonbreaking_prefix.zh │ │ └── tokenizer │ │ ├── detokenizer.perl │ │ ├── python-tokenizer │ │ └── moses.py │ │ ├── tokenizeChinese.py │ │ └── tokenizer.perl ├── models │ ├── __init__.py │ ├── base.py │ ├── dl4mt.py │ ├── mgnmt.py │ ├── rnnlm.py │ ├── rnnlm_new.py │ ├── transformer.py │ └── transformer_new.py ├── modules │ ├── __init__.py │ ├── attention.py │ ├── basic.py │ ├── cgru.py │ ├── criterions.py │ ├── embeddings.py │ ├── generator.py │ ├── position_embedding.py │ ├── rnn.py │ ├── sublayers.py │ └── variational_inferrer.py ├── optim │ ├── __init__.py │ ├── adafactor.py │ ├── adamw.py │ └── lr_scheduler.py ├── tasks │ ├── __init__.py │ ├── baseline.py │ ├── lm.py │ └── mgnmt.py └── utils │ ├── __init__.py │ ├── common_utils.py │ ├── configs.py │ ├── distributed_utils.py │ ├── init.py │ ├── logging.py │ ├── moving_average.py │ ├── nest.py │ ├── tensor_ops.py │ ├── training_utils.py │ └── vae_utils.py ├── test_mgnmt.sh ├── test_mgnmt_np.sh ├── train_mgnmt_iwslt_de2en_ed.sh ├── train_mgnmt_np_iwslt_de2en_ed.sh ├── train_mgnmt_np_partial_iwslt_de2en_ed.sh ├── train_transformer_iwslt_de2en_ed.sh ├── translate_base.sh ├── translate_base_iwslt.sh ├── translate_mgnmt.sh └── unittests ├── __init__.py ├── configs ├── test_dl4mt.yaml └── test_transformer.yaml ├── data ├── bpe.32000 ├── dev.de ├── dev.en ├── train.de ├── train.en └── vocab.bpe.32000.json ├── run_all_test.sh ├── test_dl4mt.py ├── test_transformer.py └── test_utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/.gitlab-ci.yml -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/.travis.yml -------------------------------------------------------------------------------- /.travis/install.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/.travis/install.sh -------------------------------------------------------------------------------- /BENCHMARK.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/BENCHMARK.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/LICENSE -------------------------------------------------------------------------------- /README-NJUNMT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/README-NJUNMT.md -------------------------------------------------------------------------------- /README-zh.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/README-zh.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/README.md -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/changelog.md -------------------------------------------------------------------------------- /configs/dl4mt_nist_zh2en.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/dl4mt_nist_zh2en.yaml -------------------------------------------------------------------------------- /configs/ed/iwslt_de2en_mgnmt.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/ed/iwslt_de2en_mgnmt.yaml -------------------------------------------------------------------------------- /configs/ed/iwslt_de2en_mgnmt_np.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/ed/iwslt_de2en_mgnmt_np.yaml -------------------------------------------------------------------------------- /configs/ed/iwslt_de2en_mgnmt_np_partial.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/ed/iwslt_de2en_mgnmt_np_partial.yaml -------------------------------------------------------------------------------- /configs/ed/iwslt_de2en_transformer.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/ed/iwslt_de2en_transformer.yaml -------------------------------------------------------------------------------- /configs/test_mgnmt.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/test_mgnmt.yaml -------------------------------------------------------------------------------- /configs/test_mgnmt_np.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/test_mgnmt_np.yaml -------------------------------------------------------------------------------- /configs/transformer_nist_zh2en.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/transformer_nist_zh2en.yaml -------------------------------------------------------------------------------- /configs/transformer_nist_zh2en_bpe.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/transformer_nist_zh2en_bpe.yaml -------------------------------------------------------------------------------- /configs/transformer_wmt14_en2de.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/transformer_wmt14_en2de.yaml -------------------------------------------------------------------------------- /configs/transformer_wmt15_en2fr.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/transformer_wmt15_en2fr.yaml -------------------------------------------------------------------------------- /configs/transformer_wmt16_en2ro.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/configs/transformer_wmt16_en2ro.yaml -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch>=0.4.0 2 | pyyaml 3 | tensorboardX 4 | tqdm 5 | sacrebleu -------------------------------------------------------------------------------- /scripts/build_dictionary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/scripts/build_dictionary.py -------------------------------------------------------------------------------- /scripts/checkpoints_average.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/scripts/checkpoints_average.py -------------------------------------------------------------------------------- /scripts/postprocess.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/scripts/postprocess.sh -------------------------------------------------------------------------------- /scripts/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/scripts/train.sh -------------------------------------------------------------------------------- /scripts/translate.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/scripts/translate.sh -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/bin/__init__.py -------------------------------------------------------------------------------- /src/bin/ensemble_translate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/bin/ensemble_translate.py -------------------------------------------------------------------------------- /src/bin/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/bin/train.py -------------------------------------------------------------------------------- /src/bin/translate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/bin/translate.py -------------------------------------------------------------------------------- /src/bin/translate_mgnmt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/bin/translate_mgnmt.py -------------------------------------------------------------------------------- /src/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/data/bpe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/data/bpe.py -------------------------------------------------------------------------------- /src/data/data_iterator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/data/data_iterator.py -------------------------------------------------------------------------------- /src/data/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/data/dataset.py -------------------------------------------------------------------------------- /src/data/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/data/tokenizer.py -------------------------------------------------------------------------------- /src/data/vocabulary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/data/vocabulary.py -------------------------------------------------------------------------------- /src/decoder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/decoder/__init__.py -------------------------------------------------------------------------------- /src/decoder/rnn_decoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/decoder/rnn_decoder.py -------------------------------------------------------------------------------- /src/decoder/transformer_decoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/decoder/transformer_decoder.py -------------------------------------------------------------------------------- /src/decoding/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/decoding/__init__.py -------------------------------------------------------------------------------- /src/decoding/beam_search.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/decoding/beam_search.py -------------------------------------------------------------------------------- /src/decoding/ensemble_beam_search.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/decoding/ensemble_beam_search.py -------------------------------------------------------------------------------- /src/decoding/iterative_decoding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/decoding/iterative_decoding.py -------------------------------------------------------------------------------- /src/decoding/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/decoding/utils.py -------------------------------------------------------------------------------- /src/encoder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/encoder/__init__.py -------------------------------------------------------------------------------- /src/encoder/rnn_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/encoder/rnn_encoder.py -------------------------------------------------------------------------------- /src/encoder/transformer_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/encoder/transformer_encoder.py -------------------------------------------------------------------------------- /src/metric/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/metric/bleu_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/bleu_scorer.py -------------------------------------------------------------------------------- /src/metric/scripts/generic/multi-bleu-detok.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/generic/multi-bleu-detok.perl -------------------------------------------------------------------------------- /src/metric/scripts/generic/multi-bleu.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/generic/multi-bleu.perl -------------------------------------------------------------------------------- /src/metric/scripts/recaser/detruecase.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/recaser/detruecase.perl -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/README.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/README.txt -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ca: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ca -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.cs -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.de -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.el: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.el -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.en -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.es: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.es -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fi -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fr -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ga: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ga -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.hu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.hu -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.is: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.is -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.it: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.it -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.lt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.lt -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.lv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.lv -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.nl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.nl -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pl -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pt -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ro -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ru: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ru -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sk -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sl -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sv -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ta -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.yue: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.yue -------------------------------------------------------------------------------- /src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.zh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.zh -------------------------------------------------------------------------------- /src/metric/scripts/tokenizer/detokenizer.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/tokenizer/detokenizer.perl -------------------------------------------------------------------------------- /src/metric/scripts/tokenizer/python-tokenizer/moses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/tokenizer/python-tokenizer/moses.py -------------------------------------------------------------------------------- /src/metric/scripts/tokenizer/tokenizeChinese.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/tokenizer/tokenizeChinese.py -------------------------------------------------------------------------------- /src/metric/scripts/tokenizer/tokenizer.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/metric/scripts/tokenizer/tokenizer.perl -------------------------------------------------------------------------------- /src/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/models/__init__.py -------------------------------------------------------------------------------- /src/models/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/models/base.py -------------------------------------------------------------------------------- /src/models/dl4mt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/models/dl4mt.py -------------------------------------------------------------------------------- /src/models/mgnmt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/models/mgnmt.py -------------------------------------------------------------------------------- /src/models/rnnlm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/models/rnnlm.py -------------------------------------------------------------------------------- /src/models/rnnlm_new.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/models/rnnlm_new.py -------------------------------------------------------------------------------- /src/models/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/models/transformer.py -------------------------------------------------------------------------------- /src/models/transformer_new.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/models/transformer_new.py -------------------------------------------------------------------------------- /src/modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/modules/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/modules/attention.py -------------------------------------------------------------------------------- /src/modules/basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/modules/basic.py -------------------------------------------------------------------------------- /src/modules/cgru.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/modules/cgru.py -------------------------------------------------------------------------------- /src/modules/criterions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/modules/criterions.py -------------------------------------------------------------------------------- /src/modules/embeddings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/modules/embeddings.py -------------------------------------------------------------------------------- /src/modules/generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/modules/generator.py -------------------------------------------------------------------------------- /src/modules/position_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/modules/position_embedding.py -------------------------------------------------------------------------------- /src/modules/rnn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/modules/rnn.py -------------------------------------------------------------------------------- /src/modules/sublayers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/modules/sublayers.py -------------------------------------------------------------------------------- /src/modules/variational_inferrer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/modules/variational_inferrer.py -------------------------------------------------------------------------------- /src/optim/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/optim/__init__.py -------------------------------------------------------------------------------- /src/optim/adafactor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/optim/adafactor.py -------------------------------------------------------------------------------- /src/optim/adamw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/optim/adamw.py -------------------------------------------------------------------------------- /src/optim/lr_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/optim/lr_scheduler.py -------------------------------------------------------------------------------- /src/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/tasks/__init__.py -------------------------------------------------------------------------------- /src/tasks/baseline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/tasks/baseline.py -------------------------------------------------------------------------------- /src/tasks/lm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/tasks/lm.py -------------------------------------------------------------------------------- /src/tasks/mgnmt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/tasks/mgnmt.py -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .common_utils import * 2 | -------------------------------------------------------------------------------- /src/utils/common_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/utils/common_utils.py -------------------------------------------------------------------------------- /src/utils/configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/utils/configs.py -------------------------------------------------------------------------------- /src/utils/distributed_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/utils/distributed_utils.py -------------------------------------------------------------------------------- /src/utils/init.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/utils/init.py -------------------------------------------------------------------------------- /src/utils/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/utils/logging.py -------------------------------------------------------------------------------- /src/utils/moving_average.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/utils/moving_average.py -------------------------------------------------------------------------------- /src/utils/nest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/utils/nest.py -------------------------------------------------------------------------------- /src/utils/tensor_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/utils/tensor_ops.py -------------------------------------------------------------------------------- /src/utils/training_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/utils/training_utils.py -------------------------------------------------------------------------------- /src/utils/vae_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/src/utils/vae_utils.py -------------------------------------------------------------------------------- /test_mgnmt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/test_mgnmt.sh -------------------------------------------------------------------------------- /test_mgnmt_np.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/test_mgnmt_np.sh -------------------------------------------------------------------------------- /train_mgnmt_iwslt_de2en_ed.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/train_mgnmt_iwslt_de2en_ed.sh -------------------------------------------------------------------------------- /train_mgnmt_np_iwslt_de2en_ed.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/train_mgnmt_np_iwslt_de2en_ed.sh -------------------------------------------------------------------------------- /train_mgnmt_np_partial_iwslt_de2en_ed.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/train_mgnmt_np_partial_iwslt_de2en_ed.sh -------------------------------------------------------------------------------- /train_transformer_iwslt_de2en_ed.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/train_transformer_iwslt_de2en_ed.sh -------------------------------------------------------------------------------- /translate_base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/translate_base.sh -------------------------------------------------------------------------------- /translate_base_iwslt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/translate_base_iwslt.sh -------------------------------------------------------------------------------- /translate_mgnmt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/translate_mgnmt.sh -------------------------------------------------------------------------------- /unittests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /unittests/configs/test_dl4mt.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/configs/test_dl4mt.yaml -------------------------------------------------------------------------------- /unittests/configs/test_transformer.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/configs/test_transformer.yaml -------------------------------------------------------------------------------- /unittests/data/bpe.32000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/data/bpe.32000 -------------------------------------------------------------------------------- /unittests/data/dev.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/data/dev.de -------------------------------------------------------------------------------- /unittests/data/dev.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/data/dev.en -------------------------------------------------------------------------------- /unittests/data/train.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/data/train.de -------------------------------------------------------------------------------- /unittests/data/train.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/data/train.en -------------------------------------------------------------------------------- /unittests/data/vocab.bpe.32000.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/data/vocab.bpe.32000.json -------------------------------------------------------------------------------- /unittests/run_all_test.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/run_all_test.sh -------------------------------------------------------------------------------- /unittests/test_dl4mt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/test_dl4mt.py -------------------------------------------------------------------------------- /unittests/test_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/test_transformer.py -------------------------------------------------------------------------------- /unittests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhengzx-nlp/MGNMT/HEAD/unittests/test_utils.py --------------------------------------------------------------------------------