├── .gitignore ├── README.md ├── configs ├── accelerate │ ├── gpu_1.yaml │ ├── gpu_2.yaml │ ├── gpu_3.yaml │ └── gpu_4.yaml └── routing_three_scripts_config │ └── oscar_cyrl10x_latin5x_deva13x.yaml ├── ds_configs ├── pawsx_routing_124.yml ├── pawsx_routing_51020.yml └── xnli_routing_124.yml ├── ds_scripts ├── pawsx_routing_5_10_21.sh └── xnli_routing_124.sh ├── requirements.txt ├── scripts └── run_train.sh └── src ├── data_utils.py ├── evaluation.py ├── magnet.py ├── shortening.py ├── train.py ├── train_classification.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/README.md -------------------------------------------------------------------------------- /configs/accelerate/gpu_1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/configs/accelerate/gpu_1.yaml -------------------------------------------------------------------------------- /configs/accelerate/gpu_2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/configs/accelerate/gpu_2.yaml -------------------------------------------------------------------------------- /configs/accelerate/gpu_3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/configs/accelerate/gpu_3.yaml -------------------------------------------------------------------------------- /configs/accelerate/gpu_4.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/configs/accelerate/gpu_4.yaml -------------------------------------------------------------------------------- /configs/routing_three_scripts_config/oscar_cyrl10x_latin5x_deva13x.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/configs/routing_three_scripts_config/oscar_cyrl10x_latin5x_deva13x.yaml -------------------------------------------------------------------------------- /ds_configs/pawsx_routing_124.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/ds_configs/pawsx_routing_124.yml -------------------------------------------------------------------------------- /ds_configs/pawsx_routing_51020.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/ds_configs/pawsx_routing_51020.yml -------------------------------------------------------------------------------- /ds_configs/xnli_routing_124.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/ds_configs/xnli_routing_124.yml -------------------------------------------------------------------------------- /ds_scripts/pawsx_routing_5_10_21.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/ds_scripts/pawsx_routing_5_10_21.sh -------------------------------------------------------------------------------- /ds_scripts/xnli_routing_124.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/ds_scripts/xnli_routing_124.sh -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/run_train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/scripts/run_train.sh -------------------------------------------------------------------------------- /src/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/src/data_utils.py -------------------------------------------------------------------------------- /src/evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/src/evaluation.py -------------------------------------------------------------------------------- /src/magnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/src/magnet.py -------------------------------------------------------------------------------- /src/shortening.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/src/shortening.py -------------------------------------------------------------------------------- /src/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/src/train.py -------------------------------------------------------------------------------- /src/train_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/src/train_classification.py -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orevaahia/magnet-tokenization/HEAD/src/utils.py --------------------------------------------------------------------------------