├── .gitignore ├── LICENSE ├── README.md ├── Transformer Evolution.pdf ├── albert ├── README.md ├── config.json ├── config_half.json ├── data.py ├── model.py ├── pretrain.py └── train.py ├── bert ├── README.md ├── config.json ├── config_half.json ├── data.py ├── img │ ├── accuracy.svg │ └── loss.svg ├── model.py ├── pretrain.py └── train.py ├── common_data.py ├── config.py ├── gpt ├── README.md ├── config.json ├── config_half.json ├── data.py ├── img │ ├── accuracy.svg │ ├── loss.svg │ └── object_function.png ├── model.py ├── pretrain.py └── train.py ├── img ├── accuracy.svg └── loss.svg ├── optimization.py ├── spanbert ├── README.md ├── config.json ├── config_half.json ├── data.py ├── img │ ├── accuracy.svg │ └── loss.svg ├── model.py ├── pretrain.py └── train.py ├── t5 ├── config.json ├── config_half.json ├── model.py ├── t5-01.ipynb └── t5-02.ipynb ├── transformer ├── README.md ├── config.json ├── config_half.json ├── data.py ├── img │ ├── accuracy.svg │ └── loss.svg ├── model.py └── train.py ├── tutorial ├── bert-01.ipynb ├── bert-02.ipynb ├── gpt-01.ipynb ├── gpt-02.ipynb ├── preprocess_nsmc.ipynb ├── t5_01.ipynb ├── t5_02.ipynb ├── transformer-01.ipynb ├── transformer-02.ipynb └── vocab_with_sentencepiece.ipynb └── vocab.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/README.md -------------------------------------------------------------------------------- /Transformer Evolution.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/Transformer Evolution.pdf -------------------------------------------------------------------------------- /albert/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/albert/README.md -------------------------------------------------------------------------------- /albert/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/albert/config.json -------------------------------------------------------------------------------- /albert/config_half.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/albert/config_half.json -------------------------------------------------------------------------------- /albert/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/albert/data.py -------------------------------------------------------------------------------- /albert/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/albert/model.py -------------------------------------------------------------------------------- /albert/pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/albert/pretrain.py -------------------------------------------------------------------------------- /albert/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/albert/train.py -------------------------------------------------------------------------------- /bert/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/bert/README.md -------------------------------------------------------------------------------- /bert/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/bert/config.json -------------------------------------------------------------------------------- /bert/config_half.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/bert/config_half.json -------------------------------------------------------------------------------- /bert/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/bert/data.py -------------------------------------------------------------------------------- /bert/img/accuracy.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/bert/img/accuracy.svg -------------------------------------------------------------------------------- /bert/img/loss.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/bert/img/loss.svg -------------------------------------------------------------------------------- /bert/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/bert/model.py -------------------------------------------------------------------------------- /bert/pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/bert/pretrain.py -------------------------------------------------------------------------------- /bert/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/bert/train.py -------------------------------------------------------------------------------- /common_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/common_data.py -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/config.py -------------------------------------------------------------------------------- /gpt/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/gpt/README.md -------------------------------------------------------------------------------- /gpt/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/gpt/config.json -------------------------------------------------------------------------------- /gpt/config_half.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/gpt/config_half.json -------------------------------------------------------------------------------- /gpt/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/gpt/data.py -------------------------------------------------------------------------------- /gpt/img/accuracy.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/gpt/img/accuracy.svg -------------------------------------------------------------------------------- /gpt/img/loss.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/gpt/img/loss.svg -------------------------------------------------------------------------------- /gpt/img/object_function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/gpt/img/object_function.png -------------------------------------------------------------------------------- /gpt/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/gpt/model.py -------------------------------------------------------------------------------- /gpt/pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/gpt/pretrain.py -------------------------------------------------------------------------------- /gpt/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/gpt/train.py -------------------------------------------------------------------------------- /img/accuracy.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/img/accuracy.svg -------------------------------------------------------------------------------- /img/loss.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/img/loss.svg -------------------------------------------------------------------------------- /optimization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/optimization.py -------------------------------------------------------------------------------- /spanbert/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/spanbert/README.md -------------------------------------------------------------------------------- /spanbert/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/spanbert/config.json -------------------------------------------------------------------------------- /spanbert/config_half.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/spanbert/config_half.json -------------------------------------------------------------------------------- /spanbert/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/spanbert/data.py -------------------------------------------------------------------------------- /spanbert/img/accuracy.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/spanbert/img/accuracy.svg -------------------------------------------------------------------------------- /spanbert/img/loss.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/spanbert/img/loss.svg -------------------------------------------------------------------------------- /spanbert/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/spanbert/model.py -------------------------------------------------------------------------------- /spanbert/pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/spanbert/pretrain.py -------------------------------------------------------------------------------- /spanbert/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/spanbert/train.py -------------------------------------------------------------------------------- /t5/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/t5/config.json -------------------------------------------------------------------------------- /t5/config_half.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/t5/config_half.json -------------------------------------------------------------------------------- /t5/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/t5/model.py -------------------------------------------------------------------------------- /t5/t5-01.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/t5/t5-01.ipynb -------------------------------------------------------------------------------- /t5/t5-02.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/t5/t5-02.ipynb -------------------------------------------------------------------------------- /transformer/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/transformer/README.md -------------------------------------------------------------------------------- /transformer/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/transformer/config.json -------------------------------------------------------------------------------- /transformer/config_half.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/transformer/config_half.json -------------------------------------------------------------------------------- /transformer/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/transformer/data.py -------------------------------------------------------------------------------- /transformer/img/accuracy.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/transformer/img/accuracy.svg -------------------------------------------------------------------------------- /transformer/img/loss.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/transformer/img/loss.svg -------------------------------------------------------------------------------- /transformer/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/transformer/model.py -------------------------------------------------------------------------------- /transformer/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/transformer/train.py -------------------------------------------------------------------------------- /tutorial/bert-01.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/tutorial/bert-01.ipynb -------------------------------------------------------------------------------- /tutorial/bert-02.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/tutorial/bert-02.ipynb -------------------------------------------------------------------------------- /tutorial/gpt-01.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/tutorial/gpt-01.ipynb -------------------------------------------------------------------------------- /tutorial/gpt-02.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/tutorial/gpt-02.ipynb -------------------------------------------------------------------------------- /tutorial/preprocess_nsmc.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/tutorial/preprocess_nsmc.ipynb -------------------------------------------------------------------------------- /tutorial/t5_01.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/tutorial/t5_01.ipynb -------------------------------------------------------------------------------- /tutorial/t5_02.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/tutorial/t5_02.ipynb -------------------------------------------------------------------------------- /tutorial/transformer-01.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/tutorial/transformer-01.ipynb -------------------------------------------------------------------------------- /tutorial/transformer-02.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/tutorial/transformer-02.ipynb -------------------------------------------------------------------------------- /tutorial/vocab_with_sentencepiece.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/tutorial/vocab_with_sentencepiece.ipynb -------------------------------------------------------------------------------- /vocab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paul-hyun/transformer-evolution/HEAD/vocab.py --------------------------------------------------------------------------------