├── README.md ├── ds_config1.json ├── fig ├── overmiss_example.png ├── result_swie.png └── swie_structure.png ├── infer_scripts └── run_infer.sh ├── src ├── inference.py ├── my_trainer.py ├── run_clm_llm_bloom.py ├── run_clm_llm_llama.py ├── trainable_multiL_ins_bloom.py └── trainable_multiL_ins_llama.py ├── test ├── Flores │ ├── de-en.de │ ├── de-en.en │ ├── deu_Latn.devtest │ ├── en-de.de │ ├── en-de.en │ ├── en-zh.en │ ├── en-zh.zh │ ├── eng_Latn.devtest │ ├── zh-en.en │ ├── zh-en.zh │ └── zho_Hans.devtest ├── WMT22 │ ├── concat_sents.py │ ├── newstest22.concat.de-en.de │ ├── newstest22.concat.de-en.en │ ├── newstest22.concat.en-de.de │ ├── newstest22.concat.en-de.en │ ├── newstest22.concat.en-zh.en │ ├── newstest22.concat.en-zh.zh │ ├── newstest22.concat.zh-en.en │ ├── newstest22.concat.zh-en.zh │ ├── newstest22.de-en.de │ ├── newstest22.de-en.en │ ├── newstest22.de-fr.fr │ ├── newstest22.en-de.de │ ├── newstest22.en-de.en │ ├── newstest22.en-ja.en │ ├── newstest22.en-ja.ja │ ├── newstest22.en-zh.en │ ├── newstest22.en-zh.zh │ ├── newstest22.fr-de.de │ ├── newstest22.fr-de.fr │ ├── newstest22.zh-en.en │ └── newstest22.zh-en.zh ├── instruct_inf.txt ├── instruct_inf_e2t.txt ├── instruct_inf_miss.txt ├── instruct_inf_no-error.txt ├── instruct_inf_over.txt └── instruct_inf_overmiss.txt ├── train_data └── overmiss_hf.json ├── train_scripts ├── finetune_4gpu.sh ├── finetune_4gpu_llama.sh └── finetune_8gpu.sh └── utils └── convert_alpaca_to_hf.py /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/README.md -------------------------------------------------------------------------------- /ds_config1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/ds_config1.json -------------------------------------------------------------------------------- /fig/overmiss_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/fig/overmiss_example.png -------------------------------------------------------------------------------- /fig/result_swie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/fig/result_swie.png -------------------------------------------------------------------------------- /fig/swie_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/fig/swie_structure.png -------------------------------------------------------------------------------- /infer_scripts/run_infer.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/infer_scripts/run_infer.sh -------------------------------------------------------------------------------- /src/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/src/inference.py -------------------------------------------------------------------------------- /src/my_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/src/my_trainer.py -------------------------------------------------------------------------------- /src/run_clm_llm_bloom.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/src/run_clm_llm_bloom.py -------------------------------------------------------------------------------- /src/run_clm_llm_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/src/run_clm_llm_llama.py -------------------------------------------------------------------------------- /src/trainable_multiL_ins_bloom.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/src/trainable_multiL_ins_bloom.py -------------------------------------------------------------------------------- /src/trainable_multiL_ins_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/src/trainable_multiL_ins_llama.py -------------------------------------------------------------------------------- /test/Flores/de-en.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/de-en.de -------------------------------------------------------------------------------- /test/Flores/de-en.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/de-en.en -------------------------------------------------------------------------------- /test/Flores/deu_Latn.devtest: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/deu_Latn.devtest -------------------------------------------------------------------------------- /test/Flores/en-de.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/en-de.de -------------------------------------------------------------------------------- /test/Flores/en-de.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/en-de.en -------------------------------------------------------------------------------- /test/Flores/en-zh.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/en-zh.en -------------------------------------------------------------------------------- /test/Flores/en-zh.zh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/en-zh.zh -------------------------------------------------------------------------------- /test/Flores/eng_Latn.devtest: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/eng_Latn.devtest -------------------------------------------------------------------------------- /test/Flores/zh-en.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/zh-en.en -------------------------------------------------------------------------------- /test/Flores/zh-en.zh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/zh-en.zh -------------------------------------------------------------------------------- /test/Flores/zho_Hans.devtest: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/Flores/zho_Hans.devtest -------------------------------------------------------------------------------- /test/WMT22/concat_sents.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/concat_sents.py -------------------------------------------------------------------------------- /test/WMT22/newstest22.concat.de-en.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.concat.de-en.de -------------------------------------------------------------------------------- /test/WMT22/newstest22.concat.de-en.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.concat.de-en.en -------------------------------------------------------------------------------- /test/WMT22/newstest22.concat.en-de.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.concat.en-de.de -------------------------------------------------------------------------------- /test/WMT22/newstest22.concat.en-de.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.concat.en-de.en -------------------------------------------------------------------------------- /test/WMT22/newstest22.concat.en-zh.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.concat.en-zh.en -------------------------------------------------------------------------------- /test/WMT22/newstest22.concat.en-zh.zh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.concat.en-zh.zh -------------------------------------------------------------------------------- /test/WMT22/newstest22.concat.zh-en.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.concat.zh-en.en -------------------------------------------------------------------------------- /test/WMT22/newstest22.concat.zh-en.zh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.concat.zh-en.zh -------------------------------------------------------------------------------- /test/WMT22/newstest22.de-en.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.de-en.de -------------------------------------------------------------------------------- /test/WMT22/newstest22.de-en.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.de-en.en -------------------------------------------------------------------------------- /test/WMT22/newstest22.de-fr.fr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.de-fr.fr -------------------------------------------------------------------------------- /test/WMT22/newstest22.en-de.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.en-de.de -------------------------------------------------------------------------------- /test/WMT22/newstest22.en-de.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.en-de.en -------------------------------------------------------------------------------- /test/WMT22/newstest22.en-ja.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.en-ja.en -------------------------------------------------------------------------------- /test/WMT22/newstest22.en-ja.ja: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.en-ja.ja -------------------------------------------------------------------------------- /test/WMT22/newstest22.en-zh.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.en-zh.en -------------------------------------------------------------------------------- /test/WMT22/newstest22.en-zh.zh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.en-zh.zh -------------------------------------------------------------------------------- /test/WMT22/newstest22.fr-de.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.fr-de.de -------------------------------------------------------------------------------- /test/WMT22/newstest22.fr-de.fr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.fr-de.fr -------------------------------------------------------------------------------- /test/WMT22/newstest22.zh-en.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.zh-en.en -------------------------------------------------------------------------------- /test/WMT22/newstest22.zh-en.zh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/WMT22/newstest22.zh-en.zh -------------------------------------------------------------------------------- /test/instruct_inf.txt: -------------------------------------------------------------------------------- 1 | Translate the following sentences from [SRC] to [TGT]. 2 | -------------------------------------------------------------------------------- /test/instruct_inf_e2t.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/instruct_inf_e2t.txt -------------------------------------------------------------------------------- /test/instruct_inf_miss.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/instruct_inf_miss.txt -------------------------------------------------------------------------------- /test/instruct_inf_no-error.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/instruct_inf_no-error.txt -------------------------------------------------------------------------------- /test/instruct_inf_over.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/instruct_inf_over.txt -------------------------------------------------------------------------------- /test/instruct_inf_overmiss.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/test/instruct_inf_overmiss.txt -------------------------------------------------------------------------------- /train_data/overmiss_hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/train_data/overmiss_hf.json -------------------------------------------------------------------------------- /train_scripts/finetune_4gpu.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/train_scripts/finetune_4gpu.sh -------------------------------------------------------------------------------- /train_scripts/finetune_4gpu_llama.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/train_scripts/finetune_4gpu_llama.sh -------------------------------------------------------------------------------- /train_scripts/finetune_8gpu.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/train_scripts/finetune_8gpu.sh -------------------------------------------------------------------------------- /utils/convert_alpaca_to_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pppa2019/swie_overmiss_llm4mt/HEAD/utils/convert_alpaca_to_hf.py --------------------------------------------------------------------------------