├── .github └── workflows │ ├── python-publish.yml │ └── run-tests.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── args ├── 16th_5_tkn.args ├── 30Seq.args ├── 5Seq.args ├── 60Seq.args ├── AGNews.args ├── adafactor.args ├── base.args ├── batch_large.args ├── batch_medium.args ├── batch_small.args ├── cmp_5_tkn.args ├── critic.args ├── critic_mean.args ├── cycle_loss.args ├── cycle_loss_vae.args ├── deepspeed.args ├── deepspeed_fast.args ├── deepspeed_high_cpu_ram.args ├── eigth_5_tkn.args ├── eval.args ├── full_1_tkn.args ├── full_30_tkn.args ├── full_5_tkn.args ├── funnel_medium.args ├── funnel_small.args ├── funnel_tiny.args ├── grad_accumulation_medium.args ├── grad_accumulation_small.args ├── grad_check_pnt.args ├── grad_check_pnt_enc.args ├── half_5_tkn.args ├── interpolate_batch.args ├── large_model.args ├── mnist_base.args ├── mnist_small.args ├── mnist_tiny.args ├── no_reg.args ├── qrtr_5_tkn.args ├── semantics.args ├── syntax.args ├── tenth_5_tkn.args ├── window200.args └── window60.args ├── deepspeed ├── cpu_off.json ├── no_cpu_off.json └── no_cpu_off_overlap_comm.json ├── run_experiment.py ├── setup.cfg ├── setup.py ├── t-vae.png ├── tests ├── __init__.py ├── fixtures │ ├── all_len_16.txt │ ├── empty.txt │ ├── line_by_line_max_len_3.txt │ ├── max_len_3.json │ └── multiline_max_len_4.csv └── test_train.py ├── tokenizers ├── tkn_mnist-text-small_byte │ ├── config.json │ ├── merges.txt │ ├── special_tokens_map.json │ ├── tokenizer_config.json │ └── vocab.json ├── tkn_mnist-text_byte │ ├── merges.txt │ └── vocab.json └── tkn_python-lines_byte │ ├── merges.txt │ └── vocab.json └── transformer_vae ├── __custom_t5_feedback.py ├── __init__.py ├── __main__.py ├── autoencoders.py ├── checkpoint.py ├── config.py ├── critic.py ├── custom_t5.py ├── data_collator.py ├── model.py ├── model_outputs.py ├── optimizers.py ├── sequence_checks.py ├── sklearn.py ├── tokenizer.py ├── tokenizer_train.py ├── train.py ├── trainer.py ├── trainer_callback.py └── utils.py /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/.github/workflows/python-publish.yml -------------------------------------------------------------------------------- /.github/workflows/run-tests.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/.github/workflows/run-tests.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/README.md -------------------------------------------------------------------------------- /args/16th_5_tkn.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/16th_5_tkn.args -------------------------------------------------------------------------------- /args/30Seq.args: -------------------------------------------------------------------------------- 1 | set_seq_size 30 -------------------------------------------------------------------------------- /args/5Seq.args: -------------------------------------------------------------------------------- 1 | set_seq_size 5 -------------------------------------------------------------------------------- /args/60Seq.args: -------------------------------------------------------------------------------- 1 | set_seq_size 60 -------------------------------------------------------------------------------- /args/AGNews.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/AGNews.args -------------------------------------------------------------------------------- /args/adafactor.args: -------------------------------------------------------------------------------- 1 | adafactor -------------------------------------------------------------------------------- /args/base.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/base.args -------------------------------------------------------------------------------- /args/batch_large.args: -------------------------------------------------------------------------------- 1 | per_device_train_batch_size 70 -------------------------------------------------------------------------------- /args/batch_medium.args: -------------------------------------------------------------------------------- 1 | per_device_train_batch_size 30 -------------------------------------------------------------------------------- /args/batch_small.args: -------------------------------------------------------------------------------- 1 | per_device_train_batch_size 10 -------------------------------------------------------------------------------- /args/cmp_5_tkn.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/cmp_5_tkn.args -------------------------------------------------------------------------------- /args/critic.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/critic.args -------------------------------------------------------------------------------- /args/critic_mean.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/critic_mean.args -------------------------------------------------------------------------------- /args/cycle_loss.args: -------------------------------------------------------------------------------- 1 | cycle_loss -------------------------------------------------------------------------------- /args/cycle_loss_vae.args: -------------------------------------------------------------------------------- 1 | vae_cycle_loss -------------------------------------------------------------------------------- /args/deepspeed.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/deepspeed.args -------------------------------------------------------------------------------- /args/deepspeed_fast.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/deepspeed_fast.args -------------------------------------------------------------------------------- /args/deepspeed_high_cpu_ram.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/deepspeed_high_cpu_ram.args -------------------------------------------------------------------------------- /args/eigth_5_tkn.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/eigth_5_tkn.args -------------------------------------------------------------------------------- /args/eval.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/eval.args -------------------------------------------------------------------------------- /args/full_1_tkn.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/full_1_tkn.args -------------------------------------------------------------------------------- /args/full_30_tkn.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/full_30_tkn.args -------------------------------------------------------------------------------- /args/full_5_tkn.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/full_5_tkn.args -------------------------------------------------------------------------------- /args/funnel_medium.args: -------------------------------------------------------------------------------- 1 | funnel_block_sizes=3_3_3 -------------------------------------------------------------------------------- /args/funnel_small.args: -------------------------------------------------------------------------------- 1 | funnel_block_sizes=2_2_2 -------------------------------------------------------------------------------- /args/funnel_tiny.args: -------------------------------------------------------------------------------- 1 | funnel_block_sizes=1_1_1 -------------------------------------------------------------------------------- /args/grad_accumulation_medium.args: -------------------------------------------------------------------------------- 1 | gradient_accumulation_steps 15 -------------------------------------------------------------------------------- /args/grad_accumulation_small.args: -------------------------------------------------------------------------------- 1 | gradient_accumulation_steps 2 -------------------------------------------------------------------------------- /args/grad_check_pnt.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/grad_check_pnt.args -------------------------------------------------------------------------------- /args/grad_check_pnt_enc.args: -------------------------------------------------------------------------------- 1 | gradient_checkpoint_encoder -------------------------------------------------------------------------------- /args/half_5_tkn.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/half_5_tkn.args -------------------------------------------------------------------------------- /args/interpolate_batch.args: -------------------------------------------------------------------------------- 1 | interpolate_training_step_rate 5 -------------------------------------------------------------------------------- /args/large_model.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/large_model.args -------------------------------------------------------------------------------- /args/mnist_base.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/mnist_base.args -------------------------------------------------------------------------------- /args/mnist_small.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/mnist_small.args -------------------------------------------------------------------------------- /args/mnist_tiny.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/mnist_tiny.args -------------------------------------------------------------------------------- /args/no_reg.args: -------------------------------------------------------------------------------- 1 | dont_use_reg_loss -------------------------------------------------------------------------------- /args/qrtr_5_tkn.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/qrtr_5_tkn.args -------------------------------------------------------------------------------- /args/semantics.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/semantics.args -------------------------------------------------------------------------------- /args/syntax.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/syntax.args -------------------------------------------------------------------------------- /args/tenth_5_tkn.args: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/args/tenth_5_tkn.args -------------------------------------------------------------------------------- /args/window200.args: -------------------------------------------------------------------------------- 1 | attention_window_size=200 -------------------------------------------------------------------------------- /args/window60.args: -------------------------------------------------------------------------------- 1 | attention_window_size=60 -------------------------------------------------------------------------------- /deepspeed/cpu_off.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/deepspeed/cpu_off.json -------------------------------------------------------------------------------- /deepspeed/no_cpu_off.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/deepspeed/no_cpu_off.json -------------------------------------------------------------------------------- /deepspeed/no_cpu_off_overlap_comm.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/deepspeed/no_cpu_off_overlap_comm.json -------------------------------------------------------------------------------- /run_experiment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/run_experiment.py -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/setup.cfg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/setup.py -------------------------------------------------------------------------------- /t-vae.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/t-vae.png -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/fixtures/all_len_16.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tests/fixtures/all_len_16.txt -------------------------------------------------------------------------------- /tests/fixtures/empty.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/fixtures/line_by_line_max_len_3.txt: -------------------------------------------------------------------------------- 1 | 1 2 | 22 3 | 333 4 | 4 5 | 55 6 | 666 7 | 7 8 | 88 9 | 999 -------------------------------------------------------------------------------- /tests/fixtures/max_len_3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tests/fixtures/max_len_3.json -------------------------------------------------------------------------------- /tests/fixtures/multiline_max_len_4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tests/fixtures/multiline_max_len_4.csv -------------------------------------------------------------------------------- /tests/test_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tests/test_train.py -------------------------------------------------------------------------------- /tokenizers/tkn_mnist-text-small_byte/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tokenizers/tkn_mnist-text-small_byte/config.json -------------------------------------------------------------------------------- /tokenizers/tkn_mnist-text-small_byte/merges.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tokenizers/tkn_mnist-text-small_byte/merges.txt -------------------------------------------------------------------------------- /tokenizers/tkn_mnist-text-small_byte/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tokenizers/tkn_mnist-text-small_byte/special_tokens_map.json -------------------------------------------------------------------------------- /tokenizers/tkn_mnist-text-small_byte/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tokenizers/tkn_mnist-text-small_byte/tokenizer_config.json -------------------------------------------------------------------------------- /tokenizers/tkn_mnist-text-small_byte/vocab.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tokenizers/tkn_mnist-text-small_byte/vocab.json -------------------------------------------------------------------------------- /tokenizers/tkn_mnist-text_byte/merges.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tokenizers/tkn_mnist-text_byte/merges.txt -------------------------------------------------------------------------------- /tokenizers/tkn_mnist-text_byte/vocab.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tokenizers/tkn_mnist-text_byte/vocab.json -------------------------------------------------------------------------------- /tokenizers/tkn_python-lines_byte/merges.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tokenizers/tkn_python-lines_byte/merges.txt -------------------------------------------------------------------------------- /tokenizers/tkn_python-lines_byte/vocab.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/tokenizers/tkn_python-lines_byte/vocab.json -------------------------------------------------------------------------------- /transformer_vae/__custom_t5_feedback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/__custom_t5_feedback.py -------------------------------------------------------------------------------- /transformer_vae/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/__init__.py -------------------------------------------------------------------------------- /transformer_vae/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/__main__.py -------------------------------------------------------------------------------- /transformer_vae/autoencoders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/autoencoders.py -------------------------------------------------------------------------------- /transformer_vae/checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/checkpoint.py -------------------------------------------------------------------------------- /transformer_vae/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/config.py -------------------------------------------------------------------------------- /transformer_vae/critic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/critic.py -------------------------------------------------------------------------------- /transformer_vae/custom_t5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/custom_t5.py -------------------------------------------------------------------------------- /transformer_vae/data_collator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/data_collator.py -------------------------------------------------------------------------------- /transformer_vae/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/model.py -------------------------------------------------------------------------------- /transformer_vae/model_outputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/model_outputs.py -------------------------------------------------------------------------------- /transformer_vae/optimizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/optimizers.py -------------------------------------------------------------------------------- /transformer_vae/sequence_checks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/sequence_checks.py -------------------------------------------------------------------------------- /transformer_vae/sklearn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/sklearn.py -------------------------------------------------------------------------------- /transformer_vae/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/tokenizer.py -------------------------------------------------------------------------------- /transformer_vae/tokenizer_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/tokenizer_train.py -------------------------------------------------------------------------------- /transformer_vae/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/train.py -------------------------------------------------------------------------------- /transformer_vae/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/trainer.py -------------------------------------------------------------------------------- /transformer_vae/trainer_callback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/trainer_callback.py -------------------------------------------------------------------------------- /transformer_vae/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fraser-Greenlee/transformer-vae/HEAD/transformer_vae/utils.py --------------------------------------------------------------------------------