├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── __init__.py ├── metrics_summary.py ├── model_base.py ├── model_info.py ├── optimizer.py ├── optimizer_config.py ├── requirements.txt ├── train_state.py ├── training_loop.py ├── training_task.py └── transformer ├── __init__.py ├── attention.py ├── configs ├── base_melodi.gin ├── base_memory.gin ├── base_memory2.gin ├── base_transformer.gin ├── legacy_test_set.gin ├── memory │ ├── external_memory_32k.gin │ └── external_memory_8k.gin ├── memory2 │ ├── mem_densekv_16k.gin │ ├── mem_densekv_4k.gin │ ├── mem_densekv_64k.gin │ ├── mem_densekv_txl.gin │ └── mem_none.gin ├── memory_configuration.gin ├── options │ ├── debug_floats_slow.gin │ ├── debug_mode.gin │ ├── debug_user_checks.gin │ ├── lr_cosine_decay.gin │ ├── lr_cosine_decay_spike.gin │ ├── lr_linear_decay.gin │ ├── lr_max_steps_125k.gin │ ├── lr_max_steps_250k.gin │ ├── lr_max_steps_500k.gin │ ├── lr_rsqrt_decay.gin │ ├── lr_rsqrt_decay_std.gin │ ├── lr_scale_05.gin │ ├── lr_scale_2.gin │ ├── no_norm.gin │ ├── p5x_positions_fourier.gin │ ├── positions_absolute.gin │ ├── positions_alibi.gin │ ├── positions_alibi_fixed_bound.gin │ ├── positions_fourier.gin │ ├── positions_nn.gin │ ├── positions_ortho_asinh.gin │ ├── positions_orthogonal.gin │ ├── positions_rotary.gin │ ├── positions_t5.gin │ ├── reset_pretrained_optimizer.gin │ ├── seq_1024.gin │ ├── seq_1024_vanilla.gin │ ├── seq_2048.gin │ ├── seq_2048_vanilla.gin │ ├── seq_4096.gin │ ├── seq_512.gin │ ├── seq_512_vanilla.gin │ ├── window_1024.gin │ ├── window_128.gin │ ├── window_2048.gin │ ├── window_256.gin │ └── window_512.gin ├── recurrent │ ├── bias_dual.gin │ ├── bias_single.gin │ ├── bias_skip.gin │ ├── block_feedback_512.gin │ ├── lstm_dual.gin │ ├── lstm_single.gin │ ├── lstm_skip.gin │ ├── stack_window_512.gin │ ├── states_1024.gin │ ├── states_128.gin │ ├── states_2048.gin │ ├── states_256.gin │ ├── states_512.gin │ └── tall_24L.gin ├── size │ ├── large_1200M.gin │ ├── large_600M.gin │ ├── layer13.gin │ ├── layer24.gin │ ├── layer26.gin │ ├── medium_150M.gin │ ├── medium_300M.gin │ ├── small.gin │ ├── small_37M.gin │ ├── small_75M.gin │ ├── small_test.gin │ ├── small_test_blort.gin │ └── tiny_test.gin ├── tasks │ ├── pg19_bytes.gin │ └── pg19_tokens_t5.gin └── trainer_configuration.gin ├── ht_main.py ├── inference_utils.py ├── kv_cache.py ├── language_model.py ├── launcher.py ├── memory ├── decoder_stack.py ├── memory_factory.py ├── memory_layer.py ├── transformer_base.py └── transformer_layer.py ├── memory2 ├── dense_kv_memory.py ├── melodi_layer.py ├── melodi_memory.py └── recurrent_memory_layer.py ├── metric_utils.py ├── model_runner.py ├── nn_components.py ├── position.py ├── position_alibi.py ├── position_fourier.py ├── position_nn.py ├── position_t5.py ├── synthetic_text_data.py ├── tasks.py ├── text_dataset.py ├── transformer_base.py ├── transformer_layer.py ├── transformer_stack.py ├── vocabs ├── pg19train_bpe_1000.model ├── pg19train_bpe_1000.vocab ├── pg19train_bpe_128000.model ├── pg19train_bpe_128000.vocab ├── pg19train_bpe_32000.model ├── pg19train_bpe_32000.vocab ├── pg19train_bpe_4000.model ├── pg19train_bpe_4000.vocab ├── pg19train_bpe_512.model ├── pg19train_bpe_512.vocab ├── pg19train_bpe_64000.model ├── pg19train_bpe_64000.vocab ├── pg19train_bpe_8000.model ├── pg19train_bpe_8000.vocab ├── pg19train_bpe_96000.model └── pg19train_bpe_96000.vocab └── windowed_ops.py /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/README.md -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/__init__.py -------------------------------------------------------------------------------- /metrics_summary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/metrics_summary.py -------------------------------------------------------------------------------- /model_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/model_base.py -------------------------------------------------------------------------------- /model_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/model_info.py -------------------------------------------------------------------------------- /optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/optimizer.py -------------------------------------------------------------------------------- /optimizer_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/optimizer_config.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/requirements.txt -------------------------------------------------------------------------------- /train_state.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/train_state.py -------------------------------------------------------------------------------- /training_loop.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/training_loop.py -------------------------------------------------------------------------------- /training_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/training_task.py -------------------------------------------------------------------------------- /transformer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/__init__.py -------------------------------------------------------------------------------- /transformer/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/attention.py -------------------------------------------------------------------------------- /transformer/configs/base_melodi.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/base_melodi.gin -------------------------------------------------------------------------------- /transformer/configs/base_memory.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/base_memory.gin -------------------------------------------------------------------------------- /transformer/configs/base_memory2.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/base_memory2.gin -------------------------------------------------------------------------------- /transformer/configs/base_transformer.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/base_transformer.gin -------------------------------------------------------------------------------- /transformer/configs/legacy_test_set.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/legacy_test_set.gin -------------------------------------------------------------------------------- /transformer/configs/memory/external_memory_32k.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/memory/external_memory_32k.gin -------------------------------------------------------------------------------- /transformer/configs/memory/external_memory_8k.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/memory/external_memory_8k.gin -------------------------------------------------------------------------------- /transformer/configs/memory2/mem_densekv_16k.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/memory2/mem_densekv_16k.gin -------------------------------------------------------------------------------- /transformer/configs/memory2/mem_densekv_4k.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/memory2/mem_densekv_4k.gin -------------------------------------------------------------------------------- /transformer/configs/memory2/mem_densekv_64k.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/memory2/mem_densekv_64k.gin -------------------------------------------------------------------------------- /transformer/configs/memory2/mem_densekv_txl.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/memory2/mem_densekv_txl.gin -------------------------------------------------------------------------------- /transformer/configs/memory2/mem_none.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/memory2/mem_none.gin -------------------------------------------------------------------------------- /transformer/configs/memory_configuration.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/memory_configuration.gin -------------------------------------------------------------------------------- /transformer/configs/options/debug_floats_slow.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/debug_floats_slow.gin -------------------------------------------------------------------------------- /transformer/configs/options/debug_mode.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/debug_mode.gin -------------------------------------------------------------------------------- /transformer/configs/options/debug_user_checks.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/debug_user_checks.gin -------------------------------------------------------------------------------- /transformer/configs/options/lr_cosine_decay.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/lr_cosine_decay.gin -------------------------------------------------------------------------------- /transformer/configs/options/lr_cosine_decay_spike.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/lr_cosine_decay_spike.gin -------------------------------------------------------------------------------- /transformer/configs/options/lr_linear_decay.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/lr_linear_decay.gin -------------------------------------------------------------------------------- /transformer/configs/options/lr_max_steps_125k.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/lr_max_steps_125k.gin -------------------------------------------------------------------------------- /transformer/configs/options/lr_max_steps_250k.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/lr_max_steps_250k.gin -------------------------------------------------------------------------------- /transformer/configs/options/lr_max_steps_500k.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/lr_max_steps_500k.gin -------------------------------------------------------------------------------- /transformer/configs/options/lr_rsqrt_decay.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/lr_rsqrt_decay.gin -------------------------------------------------------------------------------- /transformer/configs/options/lr_rsqrt_decay_std.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/lr_rsqrt_decay_std.gin -------------------------------------------------------------------------------- /transformer/configs/options/lr_scale_05.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/lr_scale_05.gin -------------------------------------------------------------------------------- /transformer/configs/options/lr_scale_2.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/lr_scale_2.gin -------------------------------------------------------------------------------- /transformer/configs/options/no_norm.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/no_norm.gin -------------------------------------------------------------------------------- /transformer/configs/options/p5x_positions_fourier.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/p5x_positions_fourier.gin -------------------------------------------------------------------------------- /transformer/configs/options/positions_absolute.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/positions_absolute.gin -------------------------------------------------------------------------------- /transformer/configs/options/positions_alibi.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/positions_alibi.gin -------------------------------------------------------------------------------- /transformer/configs/options/positions_alibi_fixed_bound.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/positions_alibi_fixed_bound.gin -------------------------------------------------------------------------------- /transformer/configs/options/positions_fourier.gin: -------------------------------------------------------------------------------- 1 | 2 | RELATIVE_POSITION_TYPE = "fourier" 3 | -------------------------------------------------------------------------------- /transformer/configs/options/positions_nn.gin: -------------------------------------------------------------------------------- 1 | 2 | RELATIVE_POSITION_TYPE = "nn" 3 | 4 | 5 | -------------------------------------------------------------------------------- /transformer/configs/options/positions_ortho_asinh.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/positions_ortho_asinh.gin -------------------------------------------------------------------------------- /transformer/configs/options/positions_orthogonal.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/positions_orthogonal.gin -------------------------------------------------------------------------------- /transformer/configs/options/positions_rotary.gin: -------------------------------------------------------------------------------- 1 | 2 | RELATIVE_POSITION_TYPE = "rotary" 3 | 4 | 5 | -------------------------------------------------------------------------------- /transformer/configs/options/positions_t5.gin: -------------------------------------------------------------------------------- 1 | 2 | RELATIVE_POSITION_TYPE = "t5" 3 | 4 | 5 | -------------------------------------------------------------------------------- /transformer/configs/options/reset_pretrained_optimizer.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/reset_pretrained_optimizer.gin -------------------------------------------------------------------------------- /transformer/configs/options/seq_1024.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/seq_1024.gin -------------------------------------------------------------------------------- /transformer/configs/options/seq_1024_vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/seq_1024_vanilla.gin -------------------------------------------------------------------------------- /transformer/configs/options/seq_2048.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/seq_2048.gin -------------------------------------------------------------------------------- /transformer/configs/options/seq_2048_vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/seq_2048_vanilla.gin -------------------------------------------------------------------------------- /transformer/configs/options/seq_4096.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/seq_4096.gin -------------------------------------------------------------------------------- /transformer/configs/options/seq_512.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/seq_512.gin -------------------------------------------------------------------------------- /transformer/configs/options/seq_512_vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/seq_512_vanilla.gin -------------------------------------------------------------------------------- /transformer/configs/options/window_1024.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/window_1024.gin -------------------------------------------------------------------------------- /transformer/configs/options/window_128.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/window_128.gin -------------------------------------------------------------------------------- /transformer/configs/options/window_2048.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/window_2048.gin -------------------------------------------------------------------------------- /transformer/configs/options/window_256.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/window_256.gin -------------------------------------------------------------------------------- /transformer/configs/options/window_512.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/options/window_512.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/bias_dual.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/bias_dual.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/bias_single.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/bias_single.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/bias_skip.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/bias_skip.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/block_feedback_512.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/block_feedback_512.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/lstm_dual.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/lstm_dual.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/lstm_single.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/lstm_single.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/lstm_skip.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/lstm_skip.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/stack_window_512.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/stack_window_512.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/states_1024.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/states_1024.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/states_128.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/states_128.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/states_2048.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/states_2048.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/states_256.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/states_256.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/states_512.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/states_512.gin -------------------------------------------------------------------------------- /transformer/configs/recurrent/tall_24L.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/recurrent/tall_24L.gin -------------------------------------------------------------------------------- /transformer/configs/size/large_1200M.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/large_1200M.gin -------------------------------------------------------------------------------- /transformer/configs/size/large_600M.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/large_600M.gin -------------------------------------------------------------------------------- /transformer/configs/size/layer13.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/layer13.gin -------------------------------------------------------------------------------- /transformer/configs/size/layer24.gin: -------------------------------------------------------------------------------- 1 | 2 | # Tall configuration, which doubles the number of layers and parameters. 3 | 4 | NUM_LAYERS = 24 5 | -------------------------------------------------------------------------------- /transformer/configs/size/layer26.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/layer26.gin -------------------------------------------------------------------------------- /transformer/configs/size/medium_150M.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/medium_150M.gin -------------------------------------------------------------------------------- /transformer/configs/size/medium_300M.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/medium_300M.gin -------------------------------------------------------------------------------- /transformer/configs/size/small.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/small.gin -------------------------------------------------------------------------------- /transformer/configs/size/small_37M.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/small_37M.gin -------------------------------------------------------------------------------- /transformer/configs/size/small_75M.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/small_75M.gin -------------------------------------------------------------------------------- /transformer/configs/size/small_test.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/small_test.gin -------------------------------------------------------------------------------- /transformer/configs/size/small_test_blort.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/small_test_blort.gin -------------------------------------------------------------------------------- /transformer/configs/size/tiny_test.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/size/tiny_test.gin -------------------------------------------------------------------------------- /transformer/configs/tasks/pg19_bytes.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/tasks/pg19_bytes.gin -------------------------------------------------------------------------------- /transformer/configs/tasks/pg19_tokens_t5.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/tasks/pg19_tokens_t5.gin -------------------------------------------------------------------------------- /transformer/configs/trainer_configuration.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/configs/trainer_configuration.gin -------------------------------------------------------------------------------- /transformer/ht_main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/ht_main.py -------------------------------------------------------------------------------- /transformer/inference_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/inference_utils.py -------------------------------------------------------------------------------- /transformer/kv_cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/kv_cache.py -------------------------------------------------------------------------------- /transformer/language_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/language_model.py -------------------------------------------------------------------------------- /transformer/launcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/launcher.py -------------------------------------------------------------------------------- /transformer/memory/decoder_stack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/memory/decoder_stack.py -------------------------------------------------------------------------------- /transformer/memory/memory_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/memory/memory_factory.py -------------------------------------------------------------------------------- /transformer/memory/memory_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/memory/memory_layer.py -------------------------------------------------------------------------------- /transformer/memory/transformer_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/memory/transformer_base.py -------------------------------------------------------------------------------- /transformer/memory/transformer_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/memory/transformer_layer.py -------------------------------------------------------------------------------- /transformer/memory2/dense_kv_memory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/memory2/dense_kv_memory.py -------------------------------------------------------------------------------- /transformer/memory2/melodi_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/memory2/melodi_layer.py -------------------------------------------------------------------------------- /transformer/memory2/melodi_memory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/memory2/melodi_memory.py -------------------------------------------------------------------------------- /transformer/memory2/recurrent_memory_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/memory2/recurrent_memory_layer.py -------------------------------------------------------------------------------- /transformer/metric_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/metric_utils.py -------------------------------------------------------------------------------- /transformer/model_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/model_runner.py -------------------------------------------------------------------------------- /transformer/nn_components.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/nn_components.py -------------------------------------------------------------------------------- /transformer/position.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/position.py -------------------------------------------------------------------------------- /transformer/position_alibi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/position_alibi.py -------------------------------------------------------------------------------- /transformer/position_fourier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/position_fourier.py -------------------------------------------------------------------------------- /transformer/position_nn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/position_nn.py -------------------------------------------------------------------------------- /transformer/position_t5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/position_t5.py -------------------------------------------------------------------------------- /transformer/synthetic_text_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/synthetic_text_data.py -------------------------------------------------------------------------------- /transformer/tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/tasks.py -------------------------------------------------------------------------------- /transformer/text_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/text_dataset.py -------------------------------------------------------------------------------- /transformer/transformer_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/transformer_base.py -------------------------------------------------------------------------------- /transformer/transformer_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/transformer_layer.py -------------------------------------------------------------------------------- /transformer/transformer_stack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/transformer_stack.py -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_1000.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_1000.model -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_1000.vocab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_1000.vocab -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_128000.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_128000.model -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_128000.vocab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_128000.vocab -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_32000.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_32000.model -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_32000.vocab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_32000.vocab -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_4000.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_4000.model -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_4000.vocab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_4000.vocab -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_512.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_512.model -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_512.vocab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_512.vocab -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_64000.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_64000.model -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_64000.vocab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_64000.vocab -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_8000.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_8000.model -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_8000.vocab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_8000.vocab -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_96000.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_96000.model -------------------------------------------------------------------------------- /transformer/vocabs/pg19train_bpe_96000.vocab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/vocabs/pg19train_bpe_96000.vocab -------------------------------------------------------------------------------- /transformer/windowed_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/meliad/HEAD/transformer/windowed_ops.py --------------------------------------------------------------------------------