├── .gitignore ├── .pylintrc ├── LICENSE ├── README.md ├── __init__.py ├── config_test.py ├── configs ├── __init__.py ├── config_types.py └── configs.py ├── data_preprocessing ├── __init__.py ├── bpe.py └── prepare_git_data.py ├── data_utils.py ├── distributed_eval.py ├── eval.py ├── fp16_opt.py ├── fp16util.py ├── generate_throughput_numbers.py ├── generating ├── __init__.py ├── beam_search │ ├── __init__.py │ ├── model_wrapper.py │ ├── search.py │ └── sequence_generator.py ├── generate_git.py ├── model_wrappers.py ├── sequence_generator.py ├── tokenizer_wrapper.py └── trie.py ├── get_git_data.sh ├── get_git_data_85gb.sh ├── get_wiki_data.sh ├── globals.py ├── launch.py ├── launch_tensorboard.py ├── log.py ├── loss_scaler.py ├── lr_finder.py ├── mem_transformer.py ├── prepare_repo.py ├── requirements.txt ├── scaling.png ├── shard_dataset.py ├── single_machine.sh ├── test ├── __init__.py ├── checkpoint_test.py ├── data │ ├── cache.pt │ ├── dummy_code.py │ ├── git │ │ ├── git_1.txt │ │ ├── git_2.txt │ │ ├── git_3.txt │ │ ├── git_4.txt │ │ └── train.py │ ├── test.txt │ ├── train.txt │ ├── valid.txt │ └── wikiextracted │ │ └── AA │ │ ├── wiki_00.txt │ │ ├── wiki_01.txt │ │ ├── wiki_02.txt │ │ └── wiki_03.txt └── trie.py ├── train.py ├── util.py └── utils ├── __init__.py ├── adaptive_softmax.py ├── data_parallel.py ├── exp_utils.py ├── log_uniform_sampler.py ├── proj_adaptive_softmax.py └── vocabulary.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/.gitignore -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/.pylintrc -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/README.md -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /config_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/config_test.py -------------------------------------------------------------------------------- /configs/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /configs/config_types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/configs/config_types.py -------------------------------------------------------------------------------- /configs/configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/configs/configs.py -------------------------------------------------------------------------------- /data_preprocessing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data_preprocessing/bpe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/data_preprocessing/bpe.py -------------------------------------------------------------------------------- /data_preprocessing/prepare_git_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/data_preprocessing/prepare_git_data.py -------------------------------------------------------------------------------- /data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/data_utils.py -------------------------------------------------------------------------------- /distributed_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/distributed_eval.py -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/eval.py -------------------------------------------------------------------------------- /fp16_opt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/fp16_opt.py -------------------------------------------------------------------------------- /fp16util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/fp16util.py -------------------------------------------------------------------------------- /generate_throughput_numbers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/generate_throughput_numbers.py -------------------------------------------------------------------------------- /generating/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /generating/beam_search/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /generating/beam_search/model_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/generating/beam_search/model_wrapper.py -------------------------------------------------------------------------------- /generating/beam_search/search.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/generating/beam_search/search.py -------------------------------------------------------------------------------- /generating/beam_search/sequence_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/generating/beam_search/sequence_generator.py -------------------------------------------------------------------------------- /generating/generate_git.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/generating/generate_git.py -------------------------------------------------------------------------------- /generating/model_wrappers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/generating/model_wrappers.py -------------------------------------------------------------------------------- /generating/sequence_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/generating/sequence_generator.py -------------------------------------------------------------------------------- /generating/tokenizer_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/generating/tokenizer_wrapper.py -------------------------------------------------------------------------------- /generating/trie.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/generating/trie.py -------------------------------------------------------------------------------- /get_git_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/get_git_data.sh -------------------------------------------------------------------------------- /get_git_data_85gb.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/get_git_data_85gb.sh -------------------------------------------------------------------------------- /get_wiki_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/get_wiki_data.sh -------------------------------------------------------------------------------- /globals.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/globals.py -------------------------------------------------------------------------------- /launch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/launch.py -------------------------------------------------------------------------------- /launch_tensorboard.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/launch_tensorboard.py -------------------------------------------------------------------------------- /log.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/log.py -------------------------------------------------------------------------------- /loss_scaler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/loss_scaler.py -------------------------------------------------------------------------------- /lr_finder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/lr_finder.py -------------------------------------------------------------------------------- /mem_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/mem_transformer.py -------------------------------------------------------------------------------- /prepare_repo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/prepare_repo.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/requirements.txt -------------------------------------------------------------------------------- /scaling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/scaling.png -------------------------------------------------------------------------------- /shard_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/shard_dataset.py -------------------------------------------------------------------------------- /single_machine.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/single_machine.sh -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/checkpoint_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/checkpoint_test.py -------------------------------------------------------------------------------- /test/data/cache.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/cache.pt -------------------------------------------------------------------------------- /test/data/dummy_code.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/data/git/git_1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/git/git_1.txt -------------------------------------------------------------------------------- /test/data/git/git_2.txt: -------------------------------------------------------------------------------- 1 | test_support.requires("network") 2 | -------------------------------------------------------------------------------- /test/data/git/git_3.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/git/git_3.txt -------------------------------------------------------------------------------- /test/data/git/git_4.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/git/git_4.txt -------------------------------------------------------------------------------- /test/data/git/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/git/train.py -------------------------------------------------------------------------------- /test/data/test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/test.txt -------------------------------------------------------------------------------- /test/data/train.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/train.txt -------------------------------------------------------------------------------- /test/data/valid.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/valid.txt -------------------------------------------------------------------------------- /test/data/wikiextracted/AA/wiki_00.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/wikiextracted/AA/wiki_00.txt -------------------------------------------------------------------------------- /test/data/wikiextracted/AA/wiki_01.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/wikiextracted/AA/wiki_01.txt -------------------------------------------------------------------------------- /test/data/wikiextracted/AA/wiki_02.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/wikiextracted/AA/wiki_02.txt -------------------------------------------------------------------------------- /test/data/wikiextracted/AA/wiki_03.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/data/wikiextracted/AA/wiki_03.txt -------------------------------------------------------------------------------- /test/trie.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/test/trie.py -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/train.py -------------------------------------------------------------------------------- /util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/util.py -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/adaptive_softmax.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/utils/adaptive_softmax.py -------------------------------------------------------------------------------- /utils/data_parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/utils/data_parallel.py -------------------------------------------------------------------------------- /utils/exp_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/utils/exp_utils.py -------------------------------------------------------------------------------- /utils/log_uniform_sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/utils/log_uniform_sampler.py -------------------------------------------------------------------------------- /utils/proj_adaptive_softmax.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/utils/proj_adaptive_softmax.py -------------------------------------------------------------------------------- /utils/vocabulary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybertronai/transformer-xl/HEAD/utils/vocabulary.py --------------------------------------------------------------------------------