├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── analysis ├── entropy │ ├── entropies_test_deen_base.json │ ├── entropies_test_deen_base_ls.json │ ├── entropies_test_deen_postnorm.json │ ├── entropies_test_deen_prior_3M_kl.json │ ├── entropies_test_deen_prior_3M_kl_ls.json │ ├── entropy_analysis.ipynb │ ├── entropy_bright.pdf │ ├── entropy_bright_bw=0_2.pdf │ ├── entropy_bright_postnorm.pdf │ └── entropy_bright_prior.pdf ├── results │ ├── aggregate_results.ipynb │ └── efficiency_test.pdf ├── sensitivity │ ├── figures │ │ ├── comp_bleu.test.5_lambda-tau.pdf │ │ ├── comp_bleu.test.5_lambda-tau.png │ │ ├── comp_bleu.test.5_lambda-tau_mean.pdf │ │ ├── comp_bleu.test.5_lambda-tau_mean.png │ │ ├── comp_bleu.test.5_lambda-tau_mean_std.pdf │ │ ├── comp_bleu.test.5_lambda-tau_mean_std.png │ │ ├── comp_bleu.val.5_lambda-tau.pdf │ │ ├── comp_bleu.val.5_lambda-tau.png │ │ ├── comp_bleu.val.5_lambda-tau_mean.pdf │ │ ├── comp_bleu.val.5_lambda-tau_mean.png │ │ ├── comp_bleu.val.5_lambda-tau_mean_std.pdf │ │ ├── comp_bleu.val.5_lambda-tau_mean_std.png │ │ ├── comp_lambda-tau.pdf │ │ ├── comp_lambda-tau.png │ │ ├── comp_test_lambda-tau_mean_cbar=False.pdf │ │ ├── comp_test_lambda-tau_mean_cbar=False.png │ │ ├── comp_test_lambda-tau_mean_cbar=True.pdf │ │ ├── comp_test_lambda-tau_mean_cbar=True.png │ │ ├── comp_val_lambda-tau_mean_cbar=False.pdf │ │ ├── comp_val_lambda-tau_mean_cbar=False.png │ │ ├── comp_val_lambda-tau_mean_cbar=True.pdf │ │ ├── comp_val_lambda-tau_mean_cbar=True.png │ │ ├── parallel_plot.pdf │ │ ├── parallel_plot.png │ │ └── parallel_plot.svg │ └── sensitivity_analysis.ipynb └── translation │ ├── de.txt │ ├── en.txt │ ├── extract_dists.py │ ├── index.html │ ├── rnn_samples.json │ ├── trans_samples.json │ └── trans_samples_maxlen=15.json ├── configs ├── prototype.rnn_lm_en.yaml ├── prototype.trans_lm_en.yaml ├── rnn │ ├── experiments_nmt.sh │ ├── experiments_priors.sh │ ├── generate_nmt_experiments.py │ ├── prior.lm_news_de_rnn.yaml │ ├── prior.lm_news_en_rnn.yaml │ ├── prior.lm_news_tr_rnn.yaml │ ├── rnn.deen_base.yaml │ ├── rnn.deen_fusion.yaml │ ├── rnn.deen_prior.yaml │ ├── rnn.ende_base.yaml │ ├── rnn.ende_fusion.yaml │ ├── rnn.ende_prior.yaml │ ├── rnn.entr_base.yaml │ ├── rnn.entr_fusion.yaml │ ├── rnn.entr_prior.yaml │ ├── rnn.tren_base.yaml │ ├── rnn.tren_fusion.yaml │ └── rnn.tren_prior.yaml └── transformer │ ├── experiments_nmt.sh │ ├── experiments_nmt_subsample_deen.sh │ ├── experiments_priors.sh │ ├── experiments_sensitivity.sh │ ├── generate_nmt_experiments.py │ ├── generate_sensitivity_experiments.py │ ├── prior.lm_news_de_trans.yaml │ ├── prior.lm_news_en_trans.yaml │ ├── prior.lm_news_tr_trans.yaml │ ├── trans.deen_base.yaml │ ├── trans.deen_fusion.yaml │ ├── trans.deen_prior.yaml │ ├── trans.ende_base.yaml │ ├── trans.ende_fusion.yaml │ ├── trans.ende_prior.yaml │ ├── trans.entr_base.yaml │ ├── trans.entr_fusion.yaml │ ├── trans.entr_prior.yaml │ ├── trans.tren_base.yaml │ ├── trans.tren_fusion.yaml │ └── trans.tren_prior.yaml ├── datasets ├── .gitkeep ├── mono │ ├── .gitkeep │ ├── priors │ │ └── .gitkeep │ └── prototype │ │ ├── en.train │ │ └── en.valid ├── mt │ ├── .gitkeep │ ├── download_data.sh │ ├── input-from-sgm.perl │ ├── preprocess_mono.sh │ ├── preprocess_parallel.sh │ └── subword.sh └── pp.sh ├── download_lm_checkpoints.sh ├── download_preprocessed_data.sh ├── experiments └── .gitkeep ├── fairseq_extension ├── README.md ├── __init__.py ├── data-bin │ └── .gitignore ├── data │ └── .gitignore ├── download_data.sh ├── experiments │ ├── eval-translation.sh │ ├── slurm_train_lm.sh │ └── slurm_train_mt.sh ├── prepare_data.sh ├── sys_config.py └── user │ ├── __init__.py │ └── lm_prior │ ├── __init__.py │ ├── architectures.py │ └── lm_prior.py ├── helpers ├── __init__.py ├── _logging.py ├── config.py ├── emb_utils.py ├── eval.py ├── generic.py ├── opts.py ├── text.py ├── training.py ├── transfer.py └── viz.py ├── libs ├── __init__.py └── joeynmt │ ├── __init__.py │ ├── __main__.py │ ├── attention.py │ ├── batch.py │ ├── builders.py │ ├── constants.py │ ├── data.py │ ├── decoders.py │ ├── embeddings.py │ ├── encoders.py │ ├── helpers.py │ ├── initialization.py │ ├── loss.py │ ├── metrics.py │ ├── model.py │ ├── plotting.py │ ├── prediction.py │ ├── search.py │ ├── training.py │ ├── transformer_layers.py │ └── vocabulary.py ├── model.png ├── models ├── __init__.py ├── backtranslate.py ├── nmt_prior.py ├── nmt_prior_callbacks.py ├── nmt_prior_helpers.py ├── nmt_prior_trainer.py ├── sent_lm.py ├── sent_lm_callbacks.py ├── sent_lm_trainer.py └── translate.py ├── modules ├── __init__.py ├── beam.py ├── callbacks.py ├── data │ ├── __init__.py │ ├── collates.py │ ├── datasets.py │ ├── loaders.py │ ├── samplers.py │ ├── streaming.py │ ├── utils.py │ └── vocab.py ├── helpers.py ├── initializations.py ├── layers.py ├── losses.py ├── models.py ├── modules.py ├── optim │ ├── __init__.py │ ├── lookahead.py │ ├── noam.py │ ├── radam.py │ └── ranger.py └── trainer.py ├── mylogger ├── __init__.py ├── experiment.py ├── helpers.py ├── inspection.py ├── visualization.py └── viz_samples.py ├── requirements.txt ├── sys_config.py └── tools ├── __init__.py ├── data ├── __init__.py ├── clean_parallel.py ├── filter_pairs.py ├── langid_filter.py ├── pp_mt.sh └── split_corpus.py ├── moses_scripts ├── LICENSE ├── README.md └── scripts │ ├── generic │ ├── bsbleu.py │ ├── input-from-sgm.perl │ ├── mteval-v13a.pl │ ├── mteval-v14.pl │ ├── multi-bleu-detok.perl │ ├── multi-bleu.perl │ └── wrap-xml.perl │ ├── recaser │ ├── detruecase.perl │ ├── train-truecaser.perl │ └── truecase.perl │ ├── share │ └── nonbreaking_prefixes │ │ ├── README.txt │ │ ├── nonbreaking_prefix.ca │ │ ├── nonbreaking_prefix.cs │ │ ├── nonbreaking_prefix.de │ │ ├── nonbreaking_prefix.el │ │ ├── nonbreaking_prefix.en │ │ ├── nonbreaking_prefix.es │ │ ├── nonbreaking_prefix.fi │ │ ├── nonbreaking_prefix.fr │ │ ├── nonbreaking_prefix.ga │ │ ├── nonbreaking_prefix.hu │ │ ├── nonbreaking_prefix.is │ │ ├── nonbreaking_prefix.it │ │ ├── nonbreaking_prefix.lt │ │ ├── nonbreaking_prefix.lv │ │ ├── nonbreaking_prefix.nl │ │ ├── nonbreaking_prefix.pl │ │ ├── nonbreaking_prefix.pt │ │ ├── nonbreaking_prefix.ro │ │ ├── nonbreaking_prefix.ru │ │ ├── nonbreaking_prefix.sk │ │ ├── nonbreaking_prefix.sl │ │ ├── nonbreaking_prefix.sv │ │ ├── nonbreaking_prefix.ta │ │ ├── nonbreaking_prefix.yue │ │ └── nonbreaking_prefix.zh │ ├── tokenizer │ ├── deescape-special-chars.perl │ ├── detokenizer.perl │ ├── escape-special-chars.perl │ ├── normalize-punctuation.perl │ ├── remove-non-printing-char.perl │ ├── replace-unicode-punctuation.perl │ └── tokenizer.perl │ └── training │ └── clean-corpus-n.perl └── wiki2text.sh /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/README.md -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /analysis/entropy/entropies_test_deen_base.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/entropy/entropies_test_deen_base.json -------------------------------------------------------------------------------- /analysis/entropy/entropies_test_deen_base_ls.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/entropy/entropies_test_deen_base_ls.json -------------------------------------------------------------------------------- /analysis/entropy/entropies_test_deen_postnorm.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/entropy/entropies_test_deen_postnorm.json -------------------------------------------------------------------------------- /analysis/entropy/entropies_test_deen_prior_3M_kl.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/entropy/entropies_test_deen_prior_3M_kl.json -------------------------------------------------------------------------------- /analysis/entropy/entropies_test_deen_prior_3M_kl_ls.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/entropy/entropies_test_deen_prior_3M_kl_ls.json -------------------------------------------------------------------------------- /analysis/entropy/entropy_analysis.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/entropy/entropy_analysis.ipynb -------------------------------------------------------------------------------- /analysis/entropy/entropy_bright.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/entropy/entropy_bright.pdf -------------------------------------------------------------------------------- /analysis/entropy/entropy_bright_bw=0_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/entropy/entropy_bright_bw=0_2.pdf -------------------------------------------------------------------------------- /analysis/entropy/entropy_bright_postnorm.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/entropy/entropy_bright_postnorm.pdf -------------------------------------------------------------------------------- /analysis/entropy/entropy_bright_prior.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/entropy/entropy_bright_prior.pdf -------------------------------------------------------------------------------- /analysis/results/aggregate_results.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/results/aggregate_results.ipynb -------------------------------------------------------------------------------- /analysis/results/efficiency_test.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/results/efficiency_test.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau_mean.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau_mean.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau_mean.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau_mean.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau_mean_std.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau_mean_std.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau_mean_std.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.test.5_lambda-tau_mean_std.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau_mean.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau_mean.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau_mean.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau_mean.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau_mean_std.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau_mean_std.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau_mean_std.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_bleu.val.5_lambda-tau_mean_std.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_lambda-tau.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_lambda-tau.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_lambda-tau.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_lambda-tau.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_test_lambda-tau_mean_cbar=False.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_test_lambda-tau_mean_cbar=False.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_test_lambda-tau_mean_cbar=False.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_test_lambda-tau_mean_cbar=False.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_test_lambda-tau_mean_cbar=True.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_test_lambda-tau_mean_cbar=True.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_test_lambda-tau_mean_cbar=True.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_test_lambda-tau_mean_cbar=True.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_val_lambda-tau_mean_cbar=False.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_val_lambda-tau_mean_cbar=False.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_val_lambda-tau_mean_cbar=False.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_val_lambda-tau_mean_cbar=False.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_val_lambda-tau_mean_cbar=True.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_val_lambda-tau_mean_cbar=True.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/comp_val_lambda-tau_mean_cbar=True.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/comp_val_lambda-tau_mean_cbar=True.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/parallel_plot.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/parallel_plot.pdf -------------------------------------------------------------------------------- /analysis/sensitivity/figures/parallel_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/parallel_plot.png -------------------------------------------------------------------------------- /analysis/sensitivity/figures/parallel_plot.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/figures/parallel_plot.svg -------------------------------------------------------------------------------- /analysis/sensitivity/sensitivity_analysis.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/sensitivity/sensitivity_analysis.ipynb -------------------------------------------------------------------------------- /analysis/translation/de.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/translation/de.txt -------------------------------------------------------------------------------- /analysis/translation/en.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/translation/en.txt -------------------------------------------------------------------------------- /analysis/translation/extract_dists.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/translation/extract_dists.py -------------------------------------------------------------------------------- /analysis/translation/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/translation/index.html -------------------------------------------------------------------------------- /analysis/translation/rnn_samples.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/translation/rnn_samples.json -------------------------------------------------------------------------------- /analysis/translation/trans_samples.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/translation/trans_samples.json -------------------------------------------------------------------------------- /analysis/translation/trans_samples_maxlen=15.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/analysis/translation/trans_samples_maxlen=15.json -------------------------------------------------------------------------------- /configs/prototype.rnn_lm_en.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/prototype.rnn_lm_en.yaml -------------------------------------------------------------------------------- /configs/prototype.trans_lm_en.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/prototype.trans_lm_en.yaml -------------------------------------------------------------------------------- /configs/rnn/experiments_nmt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/experiments_nmt.sh -------------------------------------------------------------------------------- /configs/rnn/experiments_priors.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/experiments_priors.sh -------------------------------------------------------------------------------- /configs/rnn/generate_nmt_experiments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/generate_nmt_experiments.py -------------------------------------------------------------------------------- /configs/rnn/prior.lm_news_de_rnn.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/prior.lm_news_de_rnn.yaml -------------------------------------------------------------------------------- /configs/rnn/prior.lm_news_en_rnn.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/prior.lm_news_en_rnn.yaml -------------------------------------------------------------------------------- /configs/rnn/prior.lm_news_tr_rnn.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/prior.lm_news_tr_rnn.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.deen_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.deen_base.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.deen_fusion.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.deen_fusion.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.deen_prior.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.deen_prior.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.ende_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.ende_base.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.ende_fusion.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.ende_fusion.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.ende_prior.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.ende_prior.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.entr_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.entr_base.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.entr_fusion.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.entr_fusion.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.entr_prior.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.entr_prior.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.tren_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.tren_base.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.tren_fusion.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.tren_fusion.yaml -------------------------------------------------------------------------------- /configs/rnn/rnn.tren_prior.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/rnn/rnn.tren_prior.yaml -------------------------------------------------------------------------------- /configs/transformer/experiments_nmt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/experiments_nmt.sh -------------------------------------------------------------------------------- /configs/transformer/experiments_nmt_subsample_deen.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/experiments_nmt_subsample_deen.sh -------------------------------------------------------------------------------- /configs/transformer/experiments_priors.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/experiments_priors.sh -------------------------------------------------------------------------------- /configs/transformer/experiments_sensitivity.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/experiments_sensitivity.sh -------------------------------------------------------------------------------- /configs/transformer/generate_nmt_experiments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/generate_nmt_experiments.py -------------------------------------------------------------------------------- /configs/transformer/generate_sensitivity_experiments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/generate_sensitivity_experiments.py -------------------------------------------------------------------------------- /configs/transformer/prior.lm_news_de_trans.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/prior.lm_news_de_trans.yaml -------------------------------------------------------------------------------- /configs/transformer/prior.lm_news_en_trans.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/prior.lm_news_en_trans.yaml -------------------------------------------------------------------------------- /configs/transformer/prior.lm_news_tr_trans.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/prior.lm_news_tr_trans.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.deen_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.deen_base.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.deen_fusion.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.deen_fusion.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.deen_prior.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.deen_prior.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.ende_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.ende_base.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.ende_fusion.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.ende_fusion.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.ende_prior.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.ende_prior.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.entr_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.entr_base.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.entr_fusion.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.entr_fusion.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.entr_prior.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.entr_prior.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.tren_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.tren_base.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.tren_fusion.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.tren_fusion.yaml -------------------------------------------------------------------------------- /configs/transformer/trans.tren_prior.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/configs/transformer/trans.tren_prior.yaml -------------------------------------------------------------------------------- /datasets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /datasets/mono/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /datasets/mono/priors/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /datasets/mono/prototype/en.train: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/datasets/mono/prototype/en.train -------------------------------------------------------------------------------- /datasets/mono/prototype/en.valid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/datasets/mono/prototype/en.valid -------------------------------------------------------------------------------- /datasets/mt/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /datasets/mt/download_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/datasets/mt/download_data.sh -------------------------------------------------------------------------------- /datasets/mt/input-from-sgm.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/datasets/mt/input-from-sgm.perl -------------------------------------------------------------------------------- /datasets/mt/preprocess_mono.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/datasets/mt/preprocess_mono.sh -------------------------------------------------------------------------------- /datasets/mt/preprocess_parallel.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/datasets/mt/preprocess_parallel.sh -------------------------------------------------------------------------------- /datasets/mt/subword.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/datasets/mt/subword.sh -------------------------------------------------------------------------------- /datasets/pp.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/datasets/pp.sh -------------------------------------------------------------------------------- /download_lm_checkpoints.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/download_lm_checkpoints.sh -------------------------------------------------------------------------------- /download_preprocessed_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/download_preprocessed_data.sh -------------------------------------------------------------------------------- /experiments/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fairseq_extension/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/fairseq_extension/README.md -------------------------------------------------------------------------------- /fairseq_extension/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fairseq_extension/data-bin/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fairseq_extension/data/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fairseq_extension/download_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/fairseq_extension/download_data.sh -------------------------------------------------------------------------------- /fairseq_extension/experiments/eval-translation.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/fairseq_extension/experiments/eval-translation.sh -------------------------------------------------------------------------------- /fairseq_extension/experiments/slurm_train_lm.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/fairseq_extension/experiments/slurm_train_lm.sh -------------------------------------------------------------------------------- /fairseq_extension/experiments/slurm_train_mt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/fairseq_extension/experiments/slurm_train_mt.sh -------------------------------------------------------------------------------- /fairseq_extension/prepare_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/fairseq_extension/prepare_data.sh -------------------------------------------------------------------------------- /fairseq_extension/sys_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/fairseq_extension/sys_config.py -------------------------------------------------------------------------------- /fairseq_extension/user/__init__.py: -------------------------------------------------------------------------------- 1 | from .lm_prior import * 2 | -------------------------------------------------------------------------------- /fairseq_extension/user/lm_prior/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/fairseq_extension/user/lm_prior/__init__.py -------------------------------------------------------------------------------- /fairseq_extension/user/lm_prior/architectures.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/fairseq_extension/user/lm_prior/architectures.py -------------------------------------------------------------------------------- /fairseq_extension/user/lm_prior/lm_prior.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/fairseq_extension/user/lm_prior/lm_prior.py -------------------------------------------------------------------------------- /helpers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/__init__.py -------------------------------------------------------------------------------- /helpers/_logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/_logging.py -------------------------------------------------------------------------------- /helpers/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/config.py -------------------------------------------------------------------------------- /helpers/emb_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/emb_utils.py -------------------------------------------------------------------------------- /helpers/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/eval.py -------------------------------------------------------------------------------- /helpers/generic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/generic.py -------------------------------------------------------------------------------- /helpers/opts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/opts.py -------------------------------------------------------------------------------- /helpers/text.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/text.py -------------------------------------------------------------------------------- /helpers/training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/training.py -------------------------------------------------------------------------------- /helpers/transfer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/transfer.py -------------------------------------------------------------------------------- /helpers/viz.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/helpers/viz.py -------------------------------------------------------------------------------- /libs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /libs/joeynmt/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /libs/joeynmt/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/__main__.py -------------------------------------------------------------------------------- /libs/joeynmt/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/attention.py -------------------------------------------------------------------------------- /libs/joeynmt/batch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/batch.py -------------------------------------------------------------------------------- /libs/joeynmt/builders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/builders.py -------------------------------------------------------------------------------- /libs/joeynmt/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/constants.py -------------------------------------------------------------------------------- /libs/joeynmt/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/data.py -------------------------------------------------------------------------------- /libs/joeynmt/decoders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/decoders.py -------------------------------------------------------------------------------- /libs/joeynmt/embeddings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/embeddings.py -------------------------------------------------------------------------------- /libs/joeynmt/encoders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/encoders.py -------------------------------------------------------------------------------- /libs/joeynmt/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/helpers.py -------------------------------------------------------------------------------- /libs/joeynmt/initialization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/initialization.py -------------------------------------------------------------------------------- /libs/joeynmt/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/loss.py -------------------------------------------------------------------------------- /libs/joeynmt/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/metrics.py -------------------------------------------------------------------------------- /libs/joeynmt/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/model.py -------------------------------------------------------------------------------- /libs/joeynmt/plotting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/plotting.py -------------------------------------------------------------------------------- /libs/joeynmt/prediction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/prediction.py -------------------------------------------------------------------------------- /libs/joeynmt/search.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/search.py -------------------------------------------------------------------------------- /libs/joeynmt/training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/training.py -------------------------------------------------------------------------------- /libs/joeynmt/transformer_layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/transformer_layers.py -------------------------------------------------------------------------------- /libs/joeynmt/vocabulary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/libs/joeynmt/vocabulary.py -------------------------------------------------------------------------------- /model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/model.png -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/models/__init__.py -------------------------------------------------------------------------------- /models/backtranslate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/models/backtranslate.py -------------------------------------------------------------------------------- /models/nmt_prior.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/models/nmt_prior.py -------------------------------------------------------------------------------- /models/nmt_prior_callbacks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/models/nmt_prior_callbacks.py -------------------------------------------------------------------------------- /models/nmt_prior_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/models/nmt_prior_helpers.py -------------------------------------------------------------------------------- /models/nmt_prior_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/models/nmt_prior_trainer.py -------------------------------------------------------------------------------- /models/sent_lm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/models/sent_lm.py -------------------------------------------------------------------------------- /models/sent_lm_callbacks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/models/sent_lm_callbacks.py -------------------------------------------------------------------------------- /models/sent_lm_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/models/sent_lm_trainer.py -------------------------------------------------------------------------------- /models/translate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/models/translate.py -------------------------------------------------------------------------------- /modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/__init__.py -------------------------------------------------------------------------------- /modules/beam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/beam.py -------------------------------------------------------------------------------- /modules/callbacks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/callbacks.py -------------------------------------------------------------------------------- /modules/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/data/__init__.py -------------------------------------------------------------------------------- /modules/data/collates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/data/collates.py -------------------------------------------------------------------------------- /modules/data/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/data/datasets.py -------------------------------------------------------------------------------- /modules/data/loaders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/data/loaders.py -------------------------------------------------------------------------------- /modules/data/samplers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/data/samplers.py -------------------------------------------------------------------------------- /modules/data/streaming.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/data/streaming.py -------------------------------------------------------------------------------- /modules/data/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/data/utils.py -------------------------------------------------------------------------------- /modules/data/vocab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/data/vocab.py -------------------------------------------------------------------------------- /modules/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/helpers.py -------------------------------------------------------------------------------- /modules/initializations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/initializations.py -------------------------------------------------------------------------------- /modules/layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/layers.py -------------------------------------------------------------------------------- /modules/losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/losses.py -------------------------------------------------------------------------------- /modules/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/models.py -------------------------------------------------------------------------------- /modules/modules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/modules.py -------------------------------------------------------------------------------- /modules/optim/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/optim/lookahead.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/optim/lookahead.py -------------------------------------------------------------------------------- /modules/optim/noam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/optim/noam.py -------------------------------------------------------------------------------- /modules/optim/radam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/optim/radam.py -------------------------------------------------------------------------------- /modules/optim/ranger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/optim/ranger.py -------------------------------------------------------------------------------- /modules/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/modules/trainer.py -------------------------------------------------------------------------------- /mylogger/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mylogger/experiment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/mylogger/experiment.py -------------------------------------------------------------------------------- /mylogger/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/mylogger/helpers.py -------------------------------------------------------------------------------- /mylogger/inspection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/mylogger/inspection.py -------------------------------------------------------------------------------- /mylogger/visualization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/mylogger/visualization.py -------------------------------------------------------------------------------- /mylogger/viz_samples.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/mylogger/viz_samples.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/requirements.txt -------------------------------------------------------------------------------- /sys_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/sys_config.py -------------------------------------------------------------------------------- /tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tools/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tools/data/clean_parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/data/clean_parallel.py -------------------------------------------------------------------------------- /tools/data/filter_pairs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/data/filter_pairs.py -------------------------------------------------------------------------------- /tools/data/langid_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/data/langid_filter.py -------------------------------------------------------------------------------- /tools/data/pp_mt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/data/pp_mt.sh -------------------------------------------------------------------------------- /tools/data/split_corpus.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/data/split_corpus.py -------------------------------------------------------------------------------- /tools/moses_scripts/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/LICENSE -------------------------------------------------------------------------------- /tools/moses_scripts/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/README.md -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/generic/bsbleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/generic/bsbleu.py -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/generic/input-from-sgm.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/generic/input-from-sgm.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/generic/mteval-v13a.pl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/generic/mteval-v13a.pl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/generic/mteval-v14.pl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/generic/mteval-v14.pl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/generic/multi-bleu-detok.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/generic/multi-bleu-detok.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/generic/multi-bleu.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/generic/multi-bleu.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/generic/wrap-xml.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/generic/wrap-xml.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/recaser/detruecase.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/recaser/detruecase.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/recaser/train-truecaser.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/recaser/train-truecaser.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/recaser/truecase.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/recaser/truecase.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/README.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/README.txt -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ca: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ca -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.cs -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.de -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.el: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.el -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.en -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.es: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.es -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fi -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fr -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ga: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ga -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.hu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.hu -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.is: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.is -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.it: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.it -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.lt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.lt -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.lv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.lv -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.nl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.nl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pt -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ro -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ru: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ru -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sk -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sv -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ta -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.yue: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.yue -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.zh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.zh -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/tokenizer/deescape-special-chars.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/tokenizer/deescape-special-chars.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/tokenizer/detokenizer.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/tokenizer/detokenizer.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/tokenizer/escape-special-chars.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/tokenizer/escape-special-chars.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/tokenizer/normalize-punctuation.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/tokenizer/normalize-punctuation.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/tokenizer/remove-non-printing-char.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/tokenizer/remove-non-printing-char.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/tokenizer/replace-unicode-punctuation.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/tokenizer/replace-unicode-punctuation.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/tokenizer/tokenizer.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/tokenizer/tokenizer.perl -------------------------------------------------------------------------------- /tools/moses_scripts/scripts/training/clean-corpus-n.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/moses_scripts/scripts/training/clean-corpus-n.perl -------------------------------------------------------------------------------- /tools/wiki2text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbaziotis/lm-prior-for-nmt/HEAD/tools/wiki2text.sh --------------------------------------------------------------------------------