├── .gitignore ├── README.md ├── consistency_testsets ├── deixis_dev.json ├── deixis_test.json ├── ellipsis_infl.json ├── ellipsis_vp.json ├── lex_cohesion_dev.json ├── lex_cohesion_test.json └── scoring_data │ ├── deixis_dev.dst │ ├── deixis_dev.src │ ├── deixis_test.dst │ ├── deixis_test.src │ ├── ellipsis_infl.dst │ ├── ellipsis_infl.src │ ├── ellipsis_vp.dst │ ├── ellipsis_vp.src │ ├── lex_cohesion_dev.dst │ ├── lex_cohesion_dev.src │ ├── lex_cohesion_test.dst │ └── lex_cohesion_test.src ├── lib ├── __init__.py ├── data.py ├── layers │ ├── __init__.py │ ├── attn.py │ └── basic.py ├── meta.py ├── ops │ ├── __init__.py │ ├── basic.py │ ├── devices.py │ ├── mpi │ │ ├── __init__.py │ │ ├── dummy_provider.py │ │ └── horovod_provider.py │ ├── record_activations.py │ └── sliced_argmax.py ├── session.py ├── task │ ├── __init__.py │ └── seq2seq │ │ ├── __init__.py │ │ ├── bleu.py │ │ ├── cadec │ │ ├── convert_checkpoint.py │ │ ├── data_to_cadec.py │ │ ├── decoder2.py │ │ ├── model.py │ │ └── problem.py │ │ ├── data.py │ │ ├── inference.py │ │ ├── models │ │ ├── DocRepair.py │ │ ├── __init__.py │ │ └── transformer.py │ │ ├── problems │ │ ├── __init__.py │ │ └── default.py │ │ ├── strutils.py │ │ ├── summary.py │ │ ├── tickers.py │ │ └── voc.py ├── tools │ ├── __init__.py │ ├── apply_bpe.py │ └── average_npz.py ├── train │ ├── __init__.py │ ├── algorithms.py │ ├── optimizers.py │ ├── problem.py │ ├── saveload.py │ └── tickers.py └── util.py ├── notebooks ├── 1_Load_model_and_translate_baseline.ipynb ├── 2_Load_model_and_translate_CADec.ipynb ├── 3_Score_consistency_test_set_baseline.ipynb └── 4_Score_consistency_test_set_CADec.ipynb ├── requirements.txt ├── resources ├── acl_emnlp_empty-min.png ├── acl_emnlp_empty.png ├── acl_empty.png ├── gif_crop_lossy100.gif ├── task_ell_vp-min.png ├── test_doc_repair-min.png ├── test_doc_repair.png ├── train_doc_repair_v_less_arrows-min.png └── train_doc_repair_v_less_arrows.png └── scripts ├── evaluate_consistency.py ├── nmt.py ├── train_baseline.sh ├── train_cadec.sh └── train_docrepair.sh /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/README.md -------------------------------------------------------------------------------- /consistency_testsets/deixis_dev.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/deixis_dev.json -------------------------------------------------------------------------------- /consistency_testsets/deixis_test.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/deixis_test.json -------------------------------------------------------------------------------- /consistency_testsets/ellipsis_infl.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/ellipsis_infl.json -------------------------------------------------------------------------------- /consistency_testsets/ellipsis_vp.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/ellipsis_vp.json -------------------------------------------------------------------------------- /consistency_testsets/lex_cohesion_dev.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/lex_cohesion_dev.json -------------------------------------------------------------------------------- /consistency_testsets/lex_cohesion_test.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/lex_cohesion_test.json -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/deixis_dev.dst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/deixis_dev.dst -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/deixis_dev.src: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/deixis_dev.src -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/deixis_test.dst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/deixis_test.dst -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/deixis_test.src: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/deixis_test.src -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/ellipsis_infl.dst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/ellipsis_infl.dst -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/ellipsis_infl.src: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/ellipsis_infl.src -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/ellipsis_vp.dst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/ellipsis_vp.dst -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/ellipsis_vp.src: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/ellipsis_vp.src -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/lex_cohesion_dev.dst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/lex_cohesion_dev.dst -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/lex_cohesion_dev.src: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/lex_cohesion_dev.src -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/lex_cohesion_test.dst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/lex_cohesion_test.dst -------------------------------------------------------------------------------- /consistency_testsets/scoring_data/lex_cohesion_test.src: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/consistency_testsets/scoring_data/lex_cohesion_test.src -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/__init__.py -------------------------------------------------------------------------------- /lib/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/data.py -------------------------------------------------------------------------------- /lib/layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/layers/__init__.py -------------------------------------------------------------------------------- /lib/layers/attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/layers/attn.py -------------------------------------------------------------------------------- /lib/layers/basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/layers/basic.py -------------------------------------------------------------------------------- /lib/meta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/meta.py -------------------------------------------------------------------------------- /lib/ops/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/ops/__init__.py -------------------------------------------------------------------------------- /lib/ops/basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/ops/basic.py -------------------------------------------------------------------------------- /lib/ops/devices.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/ops/devices.py -------------------------------------------------------------------------------- /lib/ops/mpi/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/ops/mpi/__init__.py -------------------------------------------------------------------------------- /lib/ops/mpi/dummy_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/ops/mpi/dummy_provider.py -------------------------------------------------------------------------------- /lib/ops/mpi/horovod_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/ops/mpi/horovod_provider.py -------------------------------------------------------------------------------- /lib/ops/record_activations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/ops/record_activations.py -------------------------------------------------------------------------------- /lib/ops/sliced_argmax.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/ops/sliced_argmax.py -------------------------------------------------------------------------------- /lib/session.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/session.py -------------------------------------------------------------------------------- /lib/task/__init__.py: -------------------------------------------------------------------------------- 1 | from . import seq2seq 2 | -------------------------------------------------------------------------------- /lib/task/seq2seq/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/__init__.py -------------------------------------------------------------------------------- /lib/task/seq2seq/bleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/bleu.py -------------------------------------------------------------------------------- /lib/task/seq2seq/cadec/convert_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/cadec/convert_checkpoint.py -------------------------------------------------------------------------------- /lib/task/seq2seq/cadec/data_to_cadec.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/cadec/data_to_cadec.py -------------------------------------------------------------------------------- /lib/task/seq2seq/cadec/decoder2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/cadec/decoder2.py -------------------------------------------------------------------------------- /lib/task/seq2seq/cadec/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/cadec/model.py -------------------------------------------------------------------------------- /lib/task/seq2seq/cadec/problem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/cadec/problem.py -------------------------------------------------------------------------------- /lib/task/seq2seq/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/data.py -------------------------------------------------------------------------------- /lib/task/seq2seq/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/inference.py -------------------------------------------------------------------------------- /lib/task/seq2seq/models/DocRepair.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/models/DocRepair.py -------------------------------------------------------------------------------- /lib/task/seq2seq/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/models/__init__.py -------------------------------------------------------------------------------- /lib/task/seq2seq/models/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/models/transformer.py -------------------------------------------------------------------------------- /lib/task/seq2seq/problems/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/task/seq2seq/problems/default.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/problems/default.py -------------------------------------------------------------------------------- /lib/task/seq2seq/strutils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/strutils.py -------------------------------------------------------------------------------- /lib/task/seq2seq/summary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/summary.py -------------------------------------------------------------------------------- /lib/task/seq2seq/tickers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/tickers.py -------------------------------------------------------------------------------- /lib/task/seq2seq/voc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/task/seq2seq/voc.py -------------------------------------------------------------------------------- /lib/tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/tools/apply_bpe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/tools/apply_bpe.py -------------------------------------------------------------------------------- /lib/tools/average_npz.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/tools/average_npz.py -------------------------------------------------------------------------------- /lib/train/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/train/__init__.py -------------------------------------------------------------------------------- /lib/train/algorithms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/train/algorithms.py -------------------------------------------------------------------------------- /lib/train/optimizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/train/optimizers.py -------------------------------------------------------------------------------- /lib/train/problem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/train/problem.py -------------------------------------------------------------------------------- /lib/train/saveload.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/train/saveload.py -------------------------------------------------------------------------------- /lib/train/tickers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/train/tickers.py -------------------------------------------------------------------------------- /lib/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/lib/util.py -------------------------------------------------------------------------------- /notebooks/1_Load_model_and_translate_baseline.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/notebooks/1_Load_model_and_translate_baseline.ipynb -------------------------------------------------------------------------------- /notebooks/2_Load_model_and_translate_CADec.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/notebooks/2_Load_model_and_translate_CADec.ipynb -------------------------------------------------------------------------------- /notebooks/3_Score_consistency_test_set_baseline.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/notebooks/3_Score_consistency_test_set_baseline.ipynb -------------------------------------------------------------------------------- /notebooks/4_Score_consistency_test_set_CADec.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/notebooks/4_Score_consistency_test_set_CADec.ipynb -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/requirements.txt -------------------------------------------------------------------------------- /resources/acl_emnlp_empty-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/resources/acl_emnlp_empty-min.png -------------------------------------------------------------------------------- /resources/acl_emnlp_empty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/resources/acl_emnlp_empty.png -------------------------------------------------------------------------------- /resources/acl_empty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/resources/acl_empty.png -------------------------------------------------------------------------------- /resources/gif_crop_lossy100.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/resources/gif_crop_lossy100.gif -------------------------------------------------------------------------------- /resources/task_ell_vp-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/resources/task_ell_vp-min.png -------------------------------------------------------------------------------- /resources/test_doc_repair-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/resources/test_doc_repair-min.png -------------------------------------------------------------------------------- /resources/test_doc_repair.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/resources/test_doc_repair.png -------------------------------------------------------------------------------- /resources/train_doc_repair_v_less_arrows-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/resources/train_doc_repair_v_less_arrows-min.png -------------------------------------------------------------------------------- /resources/train_doc_repair_v_less_arrows.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/resources/train_doc_repair_v_less_arrows.png -------------------------------------------------------------------------------- /scripts/evaluate_consistency.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/scripts/evaluate_consistency.py -------------------------------------------------------------------------------- /scripts/nmt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/scripts/nmt.py -------------------------------------------------------------------------------- /scripts/train_baseline.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/scripts/train_baseline.sh -------------------------------------------------------------------------------- /scripts/train_cadec.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/scripts/train_cadec.sh -------------------------------------------------------------------------------- /scripts/train_docrepair.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/good-translation-wrong-in-context/HEAD/scripts/train_docrepair.sh --------------------------------------------------------------------------------