├── .gitignore ├── README.md ├── cache_t0.sh ├── configs ├── exp │ ├── LayerNorm │ │ ├── post_layernorm.gin │ │ ├── pre_layernorm.gin │ │ ├── reset_optim.gin │ │ ├── training.gin │ │ └── training_pp.gin │ ├── PositionEmbedding │ │ ├── abs_pos.gin │ │ ├── alibi.gin │ │ ├── no_relpos.gin │ │ ├── relpos.gin │ │ └── rotary.gin │ ├── batch_size.gin │ ├── gptj.gin │ ├── memorization.gin │ ├── optim.gin │ ├── partition.gin │ ├── ratio.gin │ └── scaling.gin ├── size │ ├── 110m │ │ └── vanilla.gin │ ├── 1_6b │ │ ├── deep.gin │ │ ├── vanilla.gin │ │ └── wide.gin │ ├── 200m │ │ ├── deep.gin │ │ ├── vanilla.gin │ │ ├── vanilla_gpt.gin │ │ └── wide.gin │ ├── 25m │ │ └── vanilla.gin │ ├── 3_1b │ │ └── vanilla.gin │ ├── 470m │ │ └── vanilla.gin │ ├── 60m │ │ └── vanilla.gin │ └── 920m │ │ └── vanilla.gin ├── t5v2 │ ├── base.gin │ └── large.gin └── task │ ├── eval │ └── t0_eval.gin │ ├── finetune │ ├── codexglue │ │ ├── code_to_text_go.gin │ │ ├── code_to_text_java.gin │ │ ├── code_to_text_javascript.gin │ │ ├── code_to_text_php.gin │ │ ├── code_to_text_python.gin │ │ └── code_to_text_ruby.gin │ ├── extend_1024.gin │ ├── extend_2048-causal.gin │ ├── extend_2048-prefix.gin │ ├── extend_2048.gin │ ├── extend_4096.gin │ ├── extend_512.gin │ ├── extend_8192.gin │ ├── flan2021.gin │ ├── flan2021_t5.gin │ ├── flan2022.gin │ ├── flan2022_t5.gin │ ├── natural_sglue.gin │ ├── natural_sglue_t5.gin │ ├── pile_mlm.gin │ ├── pile_prefix_lm.gin │ ├── pile_prefix_lm_causal.gin │ ├── pile_ul2r.gin │ ├── sglue.gin │ ├── sglue_t5.gin │ └── t0_train.gin │ └── pretrain │ ├── c4_mixed_objective.gin │ ├── c4_mlm.gin │ ├── pile_causal.gin │ ├── pile_mixed_objective.gin │ ├── pile_mlm.gin │ ├── pile_mlm_causal.gin │ └── pile_prefix_lm.gin ├── convert_weights ├── configs │ ├── base_v1 │ │ ├── config.json │ │ ├── generation_config.json │ │ ├── special_tokens_map.json │ │ ├── spiece.model │ │ └── tokenizer_config.json │ ├── base_v2 │ │ ├── config.json │ │ ├── generation_config.json │ │ ├── special_tokens_map.json │ │ ├── tokenizer.model │ │ └── tokenizer_config.json │ ├── large_v1 │ │ ├── config.json │ │ ├── generation_config.json │ │ ├── special_tokens_map.json │ │ ├── spiece.model │ │ └── tokenizer_config.json │ ├── large_v2 │ │ ├── config.json │ │ ├── generation_config.json │ │ ├── special_tokens_map.json │ │ ├── tokenizer.model │ │ └── tokenizer_config.json │ ├── xl_v1 │ │ ├── config.json │ │ ├── generation_config.json │ │ ├── special_tokens_map.json │ │ ├── spiece.model │ │ └── tokenizer_config.json │ ├── xl_v2 │ │ ├── config.json │ │ ├── generation_config.json │ │ ├── special_tokens_map.json │ │ ├── tokenizer.model │ │ └── tokenizer_config.json │ ├── xxl_v1 │ │ ├── config.json │ │ ├── generation_config.json │ │ ├── special_tokens_map.json │ │ ├── spiece.model │ │ └── tokenizer_config.json │ └── xxl_v2 │ │ ├── config.json │ │ ├── generation_config.json │ │ ├── special_tokens_map.json │ │ ├── tokenizer.model │ │ └── tokenizer_config.json ├── convert-t5.txt ├── convert_t5v1_checkpoint_to_pytorch.py ├── convert_t5v2_checkpoint_to_pytorch.py ├── convert_t5x_checkpoint_to_flax.py ├── convert_t5x_checkpoint_to_pytorch.py ├── scripts │ ├── convert_v1.sh │ └── convert_v2.sh ├── upload-codexglue.sh ├── upload-multiple.sh ├── upload-t5x.sh └── upload.sh ├── data ├── __init__.py ├── bigbenchlite │ ├── __init__.py │ └── tasks.py ├── c4 │ ├── __init__.py │ ├── c4_utils.py │ └── tasks.py ├── codexglue │ ├── __init__.py │ └── tasks.py ├── flan │ ├── __init__.py │ ├── tasks.py │ └── tasks_alt.py ├── metrics.py ├── p3 │ ├── __init__.py │ └── tasks.py ├── pile │ ├── __init__.py │ ├── pile_utils.py │ └── tasks.py ├── preprocessors.py ├── preprocessors_test.py ├── sglue │ ├── __init__.py │ ├── postprocessors.py │ ├── preprocessors.py │ ├── tasks.py │ ├── tasks_natural.py │ └── tasks_t5.py ├── utils.py └── vocab.py ├── evals ├── eval-all.sh ├── eval-bbh.sh ├── eval-codexglue.sh ├── eval-cot.sh ├── eval-held_in.sh ├── eval-mmlu.sh └── eval-sglue.sh ├── experiments ├── benchmarks │ ├── t5-v1.1-lm100k │ │ ├── finetune_base_code_to_text.sh │ │ ├── finetune_large_code_to_text.sh │ │ ├── sglue_base.sh │ │ ├── sglue_large.sh │ │ ├── t0-train_base.sh │ │ └── t0-train_large.sh │ ├── t5-v1.1-lm_adapt │ │ ├── base_flan.sh │ │ ├── base_flan2021.sh │ │ ├── large_flan.sh │ │ ├── large_flan2021.sh │ │ ├── xl_flan.sh │ │ ├── xl_flan2021.sh │ │ ├── xxl_flan.sh │ │ └── xxl_flan2021.sh │ └── t5-v1.1 │ │ ├── base_code_to_text.sh │ │ ├── base_flan2021.sh │ │ ├── base_flan2022.sh │ │ ├── base_sglue.sh │ │ ├── base_t0-train.sh │ │ ├── large_code_to_text.sh │ │ ├── large_flan2021.sh │ │ ├── large_flan2022.sh │ │ ├── large_sglue.sh │ │ ├── large_t0-train.sh │ │ ├── xl_code_to_text.sh │ │ ├── xl_flan2021.sh │ │ ├── xl_flan2022.sh │ │ ├── xl_sglue.sh │ │ ├── xxl_code_to_text.sh │ │ ├── xxl_flan2021.sh │ │ ├── xxl_flan2022.sh │ │ └── xxl_sglue.sh ├── improved_t5 │ ├── ablations │ │ ├── v1-1_xl_flan2021_submix.sh │ │ └── v2_xl_flan2021_submix.sh │ ├── eval_bf16.sh │ ├── eval_fp16.sh │ ├── lm_adapt │ │ ├── finetune_base_flan.sh │ │ ├── finetune_large_flan.sh │ │ ├── finetune_xl_flan.sh │ │ ├── finetune_xxl_flan.sh │ │ ├── lm_adapt_base.sh │ │ ├── lm_adapt_large.sh │ │ ├── lm_adapt_xl.sh │ │ ├── lm_adapt_xxl.sh │ │ ├── s_causal │ │ │ ├── finetune_base_flan.sh │ │ │ ├── finetune_large_flan.sh │ │ │ ├── finetune_xl_flan.sh │ │ │ ├── finetune_xxl_flan.sh │ │ │ ├── lm_adapt_base.sh │ │ │ ├── lm_adapt_large.sh │ │ │ ├── lm_adapt_xl.sh │ │ │ └── lm_adapt_xxl.sh │ │ └── ul2r │ │ │ ├── finetune_base_ns_flan2021.sh │ │ │ ├── finetune_xl_flan2021.sh │ │ │ ├── finetune_xl_flan2022.sh │ │ │ ├── lm_adapt_base.sh │ │ │ ├── lm_adapt_base_ns.sh │ │ │ ├── lm_adapt_large.sh │ │ │ ├── lm_adapt_xl.sh │ │ │ └── lm_adapt_xxl.sh │ ├── mlm │ │ ├── extend-causal.sh │ │ ├── extend-prefix.sh │ │ ├── finetune_code_to_text.sh │ │ ├── finetune_flan2022.sh │ │ ├── finetune_sglue.sh │ │ ├── finetune_t0.sh │ │ ├── pretrain_mlm.sh │ │ └── pretrain_mlm_causal.sh │ └── ul2_causal │ │ ├── finetune_base_code_to_text.sh │ │ ├── finetune_base_flan.sh │ │ ├── finetune_base_sglue.sh │ │ ├── finetune_large_code_to_text.sh │ │ ├── finetune_large_flan.sh │ │ ├── finetune_large_sglue.sh │ │ ├── finetune_xl_code_to_text.sh │ │ ├── finetune_xl_flan2021.sh │ │ ├── finetune_xl_sglue.sh │ │ ├── finetune_xxl_flan.sh │ │ ├── finetune_xxl_sglue.sh │ │ ├── pretrain_base.sh │ │ ├── pretrain_large.sh │ │ ├── pretrain_xl.sh │ │ └── pretrain_xxl.sh └── preliminary │ ├── layernorm │ ├── eval │ │ ├── t0_eval_adamw_post.sh │ │ ├── t0_eval_adamw_post_rotary.sh │ │ ├── t0_eval_adamw_pre.sh │ │ ├── t0_eval_alibi_relpos.sh │ │ └── t0_eval_base_lm100k.sh │ ├── finetune │ │ ├── t0_train_adafactor_post.sh │ │ ├── t0_train_adafactor_pre.sh │ │ ├── t0_train_adamw_post.sh │ │ ├── t0_train_adamw_post_rotary.sh │ │ ├── t0_train_adamw_pre.sh │ │ └── t0_train_base_lm100k.sh │ ├── pretrain │ │ ├── pile_mlm_adafactor_post.sh │ │ ├── pile_mlm_adafactor_pre.sh │ │ ├── pile_mlm_adamw_post.sh │ │ ├── pile_mlm_adamw_post_rotary.sh │ │ ├── pile_mlm_adamw_pre.sh │ │ └── pile_mlm_adamw_pre_rotary.sh │ └── t0_eval.py │ ├── mixed_pretraining_objectives │ ├── finetune │ │ ├── 0_10 │ │ │ ├── sglue_finetune_920m_128000.sh │ │ │ ├── sglue_finetune_920m_160000.sh │ │ │ ├── sglue_finetune_920m_192000.sh │ │ │ ├── sglue_finetune_920m_224000.sh │ │ │ ├── sglue_finetune_920m_256000.sh │ │ │ ├── sglue_finetune_920m_32000.sh │ │ │ ├── sglue_finetune_920m_64000.sh │ │ │ └── sglue_finetune_920m_96000.sh │ │ ├── 0_15 │ │ │ ├── sglue_finetune_920m_128000.sh │ │ │ ├── sglue_finetune_920m_160000.sh │ │ │ ├── sglue_finetune_920m_192000.sh │ │ │ ├── sglue_finetune_920m_224000.sh │ │ │ ├── sglue_finetune_920m_256000.sh │ │ │ ├── sglue_finetune_920m_32000.sh │ │ │ ├── sglue_finetune_920m_64000.sh │ │ │ └── sglue_finetune_920m_96000.sh │ │ ├── 0_25 │ │ │ ├── sglue_finetune_920m_128000.sh │ │ │ ├── sglue_finetune_920m_160000.sh │ │ │ ├── sglue_finetune_920m_192000.sh │ │ │ ├── sglue_finetune_920m_224000.sh │ │ │ ├── sglue_finetune_920m_256000.sh │ │ │ ├── sglue_finetune_920m_32000.sh │ │ │ ├── sglue_finetune_920m_64000.sh │ │ │ └── sglue_finetune_920m_96000.sh │ │ ├── 0_50 │ │ │ ├── sglue_finetune_920m_128000.sh │ │ │ ├── sglue_finetune_920m_160000.sh │ │ │ ├── sglue_finetune_920m_192000.sh │ │ │ ├── sglue_finetune_920m_224000.sh │ │ │ ├── sglue_finetune_920m_256000.sh │ │ │ ├── sglue_finetune_920m_32000.sh │ │ │ ├── sglue_finetune_920m_64000.sh │ │ │ └── sglue_finetune_920m_96000.sh │ │ ├── 0_60 │ │ │ ├── sglue_finetune_920m_128000.sh │ │ │ ├── sglue_finetune_920m_160000.sh │ │ │ ├── sglue_finetune_920m_192000.sh │ │ │ ├── sglue_finetune_920m_224000.sh │ │ │ ├── sglue_finetune_920m_256000.sh │ │ │ ├── sglue_finetune_920m_32000.sh │ │ │ ├── sglue_finetune_920m_64000.sh │ │ │ └── sglue_finetune_920m_96000.sh │ │ └── 0_75 │ │ │ ├── sglue_finetune_920m_128000.sh │ │ │ ├── sglue_finetune_920m_160000.sh │ │ │ ├── sglue_finetune_920m_192000.sh │ │ │ ├── sglue_finetune_920m_224000.sh │ │ │ ├── sglue_finetune_920m_256000.sh │ │ │ ├── sglue_finetune_920m_32000.sh │ │ │ ├── sglue_finetune_920m_64000.sh │ │ │ └── sglue_finetune_920m_96000.sh │ ├── pretrain_c4 │ │ ├── 920m_c4_mlm_0_10.sh │ │ ├── 920m_c4_mlm_0_15.sh │ │ ├── 920m_c4_mlm_0_25.sh │ │ ├── 920m_c4_mlm_0_50.sh │ │ ├── 920m_c4_mlm_0_75.sh │ │ └── 920m_c4_mlm_1_00.sh │ ├── pretrain_pile │ │ ├── 920m_pile_mix_0_10.sh │ │ ├── 920m_pile_mix_0_15.sh │ │ ├── 920m_pile_mix_0_25.sh │ │ ├── 920m_pile_mix_0_50.sh │ │ ├── 920m_pile_mix_0_60.sh │ │ └── 920m_pile_mix_0_75.sh │ ├── sglue_graph.py │ ├── super_glue_performance_0-10.png │ └── super_glue_performance_flop_256k.png │ ├── partition │ ├── pretrain_pile_1.sh │ ├── pretrain_pile_2.sh │ ├── pretrain_pile_4.sh │ └── pretrain_pile_8.sh │ ├── positional_embeddings │ ├── alibi │ │ ├── eval_perplexity │ │ │ ├── no_alibi_1024_1024.sh │ │ │ ├── no_alibi_1024_114.sh │ │ │ ├── no_alibi_1024_2048.sh │ │ │ ├── no_alibi_1024_256.sh │ │ │ ├── no_alibi_1024_512.sh │ │ │ ├── no_alibi_1_1024.sh │ │ │ ├── no_alibi_1_114.sh │ │ │ ├── no_alibi_1_2048.sh │ │ │ ├── no_alibi_1_256.sh │ │ │ ├── no_alibi_1_512.sh │ │ │ ├── no_alibi_512_1024.sh │ │ │ ├── no_alibi_512_114.sh │ │ │ ├── no_alibi_512_2048.sh │ │ │ ├── no_alibi_512_256.sh │ │ │ ├── no_alibi_512_512.sh │ │ │ ├── with_alibi_1024_1024.sh │ │ │ ├── with_alibi_1024_114.sh │ │ │ ├── with_alibi_1024_2048.sh │ │ │ ├── with_alibi_1024_256.sh │ │ │ ├── with_alibi_1024_512.sh │ │ │ ├── with_alibi_1_1024.sh │ │ │ ├── with_alibi_1_114.sh │ │ │ ├── with_alibi_1_2048.sh │ │ │ ├── with_alibi_1_256.sh │ │ │ ├── with_alibi_1_512.sh │ │ │ ├── with_alibi_512_1024.sh │ │ │ ├── with_alibi_512_114.sh │ │ │ ├── with_alibi_512_2048.sh │ │ │ ├── with_alibi_512_256.sh │ │ │ └── with_alibi_512_512.sh │ │ ├── finetune_sglue_prefix_lm_no_alibi.sh │ │ ├── finetune_sglue_prefix_lm_with_alibi.sh │ │ ├── finetune_sglue_prefix_lm_with_alibi_plus_relpos.sh │ │ ├── pretrain_pile_prefix_lm_no_alibi.sh │ │ ├── pretrain_pile_prefix_lm_with_alibi.sh │ │ └── pretrain_pile_prefix_lm_with_alibi_plus_relpos.sh │ ├── pretrain_rotary_pile_mlm.sh │ └── rotary │ │ ├── eval │ │ ├── t0_eval_alibi_relpos.sh │ │ ├── t0_eval_benchmark.sh │ │ ├── t0_eval_metro_learning_only_training.sh │ │ ├── t0_eval_metro_no_alibi.sh │ │ └── t0_eval_metro_training.sh │ │ ├── finetune │ │ └── sglue │ │ │ ├── sglue_train_benchmark.sh │ │ │ ├── sglue_train_rotary.sh │ │ │ └── sglue_train_rotary_relpos.sh │ │ ├── pretrain_benchmark_pile_mlm.sh │ │ ├── pretrain_rotary_relpos_pile_mlm.sh │ │ ├── sglue_graph.py │ │ └── t0_eval.py │ └── scaling_laws │ ├── 110m │ ├── sglue_finetune_110m_16000.sh │ ├── sglue_finetune_110m_32000.sh │ ├── sglue_finetune_110m_48000.sh │ ├── sglue_finetune_110m_64000.sh │ └── sglue_finetune_110m_80000.sh │ ├── 1_6b │ ├── sglue_finetune_1_6b_128000.sh │ ├── sglue_finetune_1_6b_192000.sh │ ├── sglue_finetune_1_6b_256000.sh │ ├── sglue_finetune_1_6b_320000.sh │ ├── sglue_finetune_1_6b_384000.sh │ ├── sglue_finetune_1_6b_424000.sh │ ├── sglue_finetune_1_6b_448000.sh │ ├── sglue_finetune_1_6b_512000.sh │ └── sglue_finetune_1_6b_64000.sh │ ├── 25m │ ├── sglue_finetune_25m_16000.sh │ ├── sglue_finetune_25m_24000.sh │ ├── sglue_finetune_25m_32000.sh │ ├── sglue_finetune_25m_40000.sh │ ├── sglue_finetune_25m_48000.sh │ ├── sglue_finetune_25m_56000.sh │ ├── sglue_finetune_25m_64000.sh │ └── sglue_finetune_25m_8000.sh │ ├── 920m │ ├── pretrain_c4.sh │ ├── pretrain_pile.sh │ ├── sglue_finetune_920m_128000.sh │ ├── sglue_finetune_920m_160000.sh │ ├── sglue_finetune_920m_192000.sh │ ├── sglue_finetune_920m_224000.sh │ ├── sglue_finetune_920m_256000.sh │ ├── sglue_finetune_920m_32000.sh │ ├── sglue_finetune_920m_64000.sh │ └── sglue_finetune_920m_96000.sh │ ├── sglue_finetune.sh │ ├── sglue_graph.py │ ├── sglue_graph_.py │ ├── super_glue_performance.png │ └── super_glue_performance_flop.png ├── models ├── decoder_t5 │ ├── __init__.py │ └── modeling_decoder_t5.py └── scalable_t5 │ ├── README.md │ ├── __init__.py │ ├── alibi_position_biases.py │ ├── alibi_position_biases_test.py │ ├── layers.py │ ├── layers_test.py │ ├── local_tiny.gin │ ├── mt5 │ ├── __init__.py │ ├── base.gin │ ├── large.gin │ ├── small.gin │ ├── xl.gin │ └── xxl.gin │ ├── network.py │ ├── network_test.py │ ├── rotary_embedding.py │ ├── rotary_embedding_test.py │ └── t5_1_1 │ ├── __init__.py │ ├── base.gin │ ├── examples │ ├── __init__.py │ └── wmt19_ende_from_scratch.gin │ ├── large.gin │ ├── small.gin │ ├── xl.gin │ └── xxl.gin ├── setup.py └── tpu-scripts ├── kill.sh ├── run.sh ├── send.sh └── setup.sh /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/README.md -------------------------------------------------------------------------------- /cache_t0.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/cache_t0.sh -------------------------------------------------------------------------------- /configs/exp/LayerNorm/post_layernorm.gin: -------------------------------------------------------------------------------- 1 | network.T5Config.pre_layer_norm = False -------------------------------------------------------------------------------- /configs/exp/LayerNorm/pre_layernorm.gin: -------------------------------------------------------------------------------- 1 | network.T5Config.pre_layer_norm = True 2 | -------------------------------------------------------------------------------- /configs/exp/LayerNorm/reset_optim.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/exp/LayerNorm/reset_optim.gin -------------------------------------------------------------------------------- /configs/exp/LayerNorm/training.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/exp/LayerNorm/training.gin -------------------------------------------------------------------------------- /configs/exp/LayerNorm/training_pp.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/exp/LayerNorm/training_pp.gin -------------------------------------------------------------------------------- /configs/exp/PositionEmbedding/abs_pos.gin: -------------------------------------------------------------------------------- 1 | network.T5Config.use_abs_pos_embedding = True -------------------------------------------------------------------------------- /configs/exp/PositionEmbedding/alibi.gin: -------------------------------------------------------------------------------- 1 | network.T5Config.use_alibi = True -------------------------------------------------------------------------------- /configs/exp/PositionEmbedding/no_relpos.gin: -------------------------------------------------------------------------------- 1 | network.T5Config.use_rel_pos = False -------------------------------------------------------------------------------- /configs/exp/PositionEmbedding/relpos.gin: -------------------------------------------------------------------------------- 1 | network.T5Config.use_rel_pos = True -------------------------------------------------------------------------------- /configs/exp/PositionEmbedding/rotary.gin: -------------------------------------------------------------------------------- 1 | network.T5Config.use_rotary_embedding = True -------------------------------------------------------------------------------- /configs/exp/batch_size.gin: -------------------------------------------------------------------------------- 1 | utils.SaveCheckpointConfig.keep = 1 -------------------------------------------------------------------------------- /configs/exp/gptj.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/exp/gptj.gin -------------------------------------------------------------------------------- /configs/exp/memorization.gin: -------------------------------------------------------------------------------- 1 | BATCH_SIZE = 2048 2 | 3 | utils.SaveCheckpointConfig.keep = 1 -------------------------------------------------------------------------------- /configs/exp/optim.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/exp/optim.gin -------------------------------------------------------------------------------- /configs/exp/partition.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/exp/partition.gin -------------------------------------------------------------------------------- /configs/exp/ratio.gin: -------------------------------------------------------------------------------- 1 | BATCH_SIZE = 2048 2 | 3 | utils.SaveCheckpointConfig.keep = 1 -------------------------------------------------------------------------------- /configs/exp/scaling.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/exp/scaling.gin -------------------------------------------------------------------------------- /configs/size/110m/vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/110m/vanilla.gin -------------------------------------------------------------------------------- /configs/size/1_6b/deep.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/1_6b/deep.gin -------------------------------------------------------------------------------- /configs/size/1_6b/vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/1_6b/vanilla.gin -------------------------------------------------------------------------------- /configs/size/1_6b/wide.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/1_6b/wide.gin -------------------------------------------------------------------------------- /configs/size/200m/deep.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/200m/deep.gin -------------------------------------------------------------------------------- /configs/size/200m/vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/200m/vanilla.gin -------------------------------------------------------------------------------- /configs/size/200m/vanilla_gpt.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/200m/vanilla_gpt.gin -------------------------------------------------------------------------------- /configs/size/200m/wide.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/200m/wide.gin -------------------------------------------------------------------------------- /configs/size/25m/vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/25m/vanilla.gin -------------------------------------------------------------------------------- /configs/size/3_1b/vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/3_1b/vanilla.gin -------------------------------------------------------------------------------- /configs/size/470m/vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/470m/vanilla.gin -------------------------------------------------------------------------------- /configs/size/60m/vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/60m/vanilla.gin -------------------------------------------------------------------------------- /configs/size/920m/vanilla.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/size/920m/vanilla.gin -------------------------------------------------------------------------------- /configs/t5v2/base.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/t5v2/base.gin -------------------------------------------------------------------------------- /configs/t5v2/large.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/t5v2/large.gin -------------------------------------------------------------------------------- /configs/task/eval/t0_eval.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/eval/t0_eval.gin -------------------------------------------------------------------------------- /configs/task/finetune/codexglue/code_to_text_go.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/codexglue/code_to_text_go.gin -------------------------------------------------------------------------------- /configs/task/finetune/codexglue/code_to_text_java.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/codexglue/code_to_text_java.gin -------------------------------------------------------------------------------- /configs/task/finetune/codexglue/code_to_text_javascript.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/codexglue/code_to_text_javascript.gin -------------------------------------------------------------------------------- /configs/task/finetune/codexglue/code_to_text_php.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/codexglue/code_to_text_php.gin -------------------------------------------------------------------------------- /configs/task/finetune/codexglue/code_to_text_python.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/codexglue/code_to_text_python.gin -------------------------------------------------------------------------------- /configs/task/finetune/codexglue/code_to_text_ruby.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/codexglue/code_to_text_ruby.gin -------------------------------------------------------------------------------- /configs/task/finetune/extend_1024.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/extend_1024.gin -------------------------------------------------------------------------------- /configs/task/finetune/extend_2048-causal.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/extend_2048-causal.gin -------------------------------------------------------------------------------- /configs/task/finetune/extend_2048-prefix.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/extend_2048-prefix.gin -------------------------------------------------------------------------------- /configs/task/finetune/extend_2048.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/extend_2048.gin -------------------------------------------------------------------------------- /configs/task/finetune/extend_4096.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/extend_4096.gin -------------------------------------------------------------------------------- /configs/task/finetune/extend_512.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/extend_512.gin -------------------------------------------------------------------------------- /configs/task/finetune/extend_8192.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/extend_8192.gin -------------------------------------------------------------------------------- /configs/task/finetune/flan2021.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/flan2021.gin -------------------------------------------------------------------------------- /configs/task/finetune/flan2021_t5.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/flan2021_t5.gin -------------------------------------------------------------------------------- /configs/task/finetune/flan2022.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/flan2022.gin -------------------------------------------------------------------------------- /configs/task/finetune/flan2022_t5.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/flan2022_t5.gin -------------------------------------------------------------------------------- /configs/task/finetune/natural_sglue.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/natural_sglue.gin -------------------------------------------------------------------------------- /configs/task/finetune/natural_sglue_t5.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/natural_sglue_t5.gin -------------------------------------------------------------------------------- /configs/task/finetune/pile_mlm.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/pile_mlm.gin -------------------------------------------------------------------------------- /configs/task/finetune/pile_prefix_lm.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/pile_prefix_lm.gin -------------------------------------------------------------------------------- /configs/task/finetune/pile_prefix_lm_causal.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/pile_prefix_lm_causal.gin -------------------------------------------------------------------------------- /configs/task/finetune/pile_ul2r.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/pile_ul2r.gin -------------------------------------------------------------------------------- /configs/task/finetune/sglue.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/sglue.gin -------------------------------------------------------------------------------- /configs/task/finetune/sglue_t5.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/sglue_t5.gin -------------------------------------------------------------------------------- /configs/task/finetune/t0_train.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/finetune/t0_train.gin -------------------------------------------------------------------------------- /configs/task/pretrain/c4_mixed_objective.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/pretrain/c4_mixed_objective.gin -------------------------------------------------------------------------------- /configs/task/pretrain/c4_mlm.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/pretrain/c4_mlm.gin -------------------------------------------------------------------------------- /configs/task/pretrain/pile_causal.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/pretrain/pile_causal.gin -------------------------------------------------------------------------------- /configs/task/pretrain/pile_mixed_objective.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/pretrain/pile_mixed_objective.gin -------------------------------------------------------------------------------- /configs/task/pretrain/pile_mlm.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/pretrain/pile_mlm.gin -------------------------------------------------------------------------------- /configs/task/pretrain/pile_mlm_causal.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/pretrain/pile_mlm_causal.gin -------------------------------------------------------------------------------- /configs/task/pretrain/pile_prefix_lm.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/configs/task/pretrain/pile_prefix_lm.gin -------------------------------------------------------------------------------- /convert_weights/configs/base_v1/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/base_v1/config.json -------------------------------------------------------------------------------- /convert_weights/configs/base_v1/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/base_v1/generation_config.json -------------------------------------------------------------------------------- /convert_weights/configs/base_v1/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/base_v1/special_tokens_map.json -------------------------------------------------------------------------------- /convert_weights/configs/base_v1/spiece.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/base_v1/spiece.model -------------------------------------------------------------------------------- /convert_weights/configs/base_v1/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/base_v1/tokenizer_config.json -------------------------------------------------------------------------------- /convert_weights/configs/base_v2/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/base_v2/config.json -------------------------------------------------------------------------------- /convert_weights/configs/base_v2/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/base_v2/generation_config.json -------------------------------------------------------------------------------- /convert_weights/configs/base_v2/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/base_v2/special_tokens_map.json -------------------------------------------------------------------------------- /convert_weights/configs/base_v2/tokenizer.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/base_v2/tokenizer.model -------------------------------------------------------------------------------- /convert_weights/configs/base_v2/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/base_v2/tokenizer_config.json -------------------------------------------------------------------------------- /convert_weights/configs/large_v1/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/large_v1/config.json -------------------------------------------------------------------------------- /convert_weights/configs/large_v1/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/large_v1/generation_config.json -------------------------------------------------------------------------------- /convert_weights/configs/large_v1/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/large_v1/special_tokens_map.json -------------------------------------------------------------------------------- /convert_weights/configs/large_v1/spiece.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/large_v1/spiece.model -------------------------------------------------------------------------------- /convert_weights/configs/large_v1/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/large_v1/tokenizer_config.json -------------------------------------------------------------------------------- /convert_weights/configs/large_v2/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/large_v2/config.json -------------------------------------------------------------------------------- /convert_weights/configs/large_v2/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/large_v2/generation_config.json -------------------------------------------------------------------------------- /convert_weights/configs/large_v2/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/large_v2/special_tokens_map.json -------------------------------------------------------------------------------- /convert_weights/configs/large_v2/tokenizer.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/large_v2/tokenizer.model -------------------------------------------------------------------------------- /convert_weights/configs/large_v2/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/large_v2/tokenizer_config.json -------------------------------------------------------------------------------- /convert_weights/configs/xl_v1/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xl_v1/config.json -------------------------------------------------------------------------------- /convert_weights/configs/xl_v1/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xl_v1/generation_config.json -------------------------------------------------------------------------------- /convert_weights/configs/xl_v1/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xl_v1/special_tokens_map.json -------------------------------------------------------------------------------- /convert_weights/configs/xl_v1/spiece.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xl_v1/spiece.model -------------------------------------------------------------------------------- /convert_weights/configs/xl_v1/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xl_v1/tokenizer_config.json -------------------------------------------------------------------------------- /convert_weights/configs/xl_v2/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xl_v2/config.json -------------------------------------------------------------------------------- /convert_weights/configs/xl_v2/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xl_v2/generation_config.json -------------------------------------------------------------------------------- /convert_weights/configs/xl_v2/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xl_v2/special_tokens_map.json -------------------------------------------------------------------------------- /convert_weights/configs/xl_v2/tokenizer.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xl_v2/tokenizer.model -------------------------------------------------------------------------------- /convert_weights/configs/xl_v2/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xl_v2/tokenizer_config.json -------------------------------------------------------------------------------- /convert_weights/configs/xxl_v1/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xxl_v1/config.json -------------------------------------------------------------------------------- /convert_weights/configs/xxl_v1/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xxl_v1/generation_config.json -------------------------------------------------------------------------------- /convert_weights/configs/xxl_v1/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xxl_v1/special_tokens_map.json -------------------------------------------------------------------------------- /convert_weights/configs/xxl_v1/spiece.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xxl_v1/spiece.model -------------------------------------------------------------------------------- /convert_weights/configs/xxl_v1/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xxl_v1/tokenizer_config.json -------------------------------------------------------------------------------- /convert_weights/configs/xxl_v2/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xxl_v2/config.json -------------------------------------------------------------------------------- /convert_weights/configs/xxl_v2/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xxl_v2/generation_config.json -------------------------------------------------------------------------------- /convert_weights/configs/xxl_v2/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xxl_v2/special_tokens_map.json -------------------------------------------------------------------------------- /convert_weights/configs/xxl_v2/tokenizer.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xxl_v2/tokenizer.model -------------------------------------------------------------------------------- /convert_weights/configs/xxl_v2/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/configs/xxl_v2/tokenizer_config.json -------------------------------------------------------------------------------- /convert_weights/convert-t5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/convert-t5.txt -------------------------------------------------------------------------------- /convert_weights/convert_t5v1_checkpoint_to_pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/convert_t5v1_checkpoint_to_pytorch.py -------------------------------------------------------------------------------- /convert_weights/convert_t5v2_checkpoint_to_pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/convert_t5v2_checkpoint_to_pytorch.py -------------------------------------------------------------------------------- /convert_weights/convert_t5x_checkpoint_to_flax.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/convert_t5x_checkpoint_to_flax.py -------------------------------------------------------------------------------- /convert_weights/convert_t5x_checkpoint_to_pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/convert_t5x_checkpoint_to_pytorch.py -------------------------------------------------------------------------------- /convert_weights/scripts/convert_v1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/scripts/convert_v1.sh -------------------------------------------------------------------------------- /convert_weights/scripts/convert_v2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/scripts/convert_v2.sh -------------------------------------------------------------------------------- /convert_weights/upload-codexglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/upload-codexglue.sh -------------------------------------------------------------------------------- /convert_weights/upload-multiple.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/upload-multiple.sh -------------------------------------------------------------------------------- /convert_weights/upload-t5x.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/upload-t5x.sh -------------------------------------------------------------------------------- /convert_weights/upload.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/convert_weights/upload.sh -------------------------------------------------------------------------------- /data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/__init__.py -------------------------------------------------------------------------------- /data/bigbenchlite/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/bigbenchlite/tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/bigbenchlite/tasks.py -------------------------------------------------------------------------------- /data/c4/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/c4/c4_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/c4/c4_utils.py -------------------------------------------------------------------------------- /data/c4/tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/c4/tasks.py -------------------------------------------------------------------------------- /data/codexglue/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/codexglue/tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/codexglue/tasks.py -------------------------------------------------------------------------------- /data/flan/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/flan/tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/flan/tasks.py -------------------------------------------------------------------------------- /data/flan/tasks_alt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/flan/tasks_alt.py -------------------------------------------------------------------------------- /data/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/metrics.py -------------------------------------------------------------------------------- /data/p3/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/p3/tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/p3/tasks.py -------------------------------------------------------------------------------- /data/pile/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/pile/pile_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/pile/pile_utils.py -------------------------------------------------------------------------------- /data/pile/tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/pile/tasks.py -------------------------------------------------------------------------------- /data/preprocessors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/preprocessors.py -------------------------------------------------------------------------------- /data/preprocessors_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/preprocessors_test.py -------------------------------------------------------------------------------- /data/sglue/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/sglue/postprocessors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/sglue/postprocessors.py -------------------------------------------------------------------------------- /data/sglue/preprocessors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/sglue/preprocessors.py -------------------------------------------------------------------------------- /data/sglue/tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/sglue/tasks.py -------------------------------------------------------------------------------- /data/sglue/tasks_natural.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/sglue/tasks_natural.py -------------------------------------------------------------------------------- /data/sglue/tasks_t5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/sglue/tasks_t5.py -------------------------------------------------------------------------------- /data/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/utils.py -------------------------------------------------------------------------------- /data/vocab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/data/vocab.py -------------------------------------------------------------------------------- /evals/eval-all.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/evals/eval-all.sh -------------------------------------------------------------------------------- /evals/eval-bbh.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/evals/eval-bbh.sh -------------------------------------------------------------------------------- /evals/eval-codexglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/evals/eval-codexglue.sh -------------------------------------------------------------------------------- /evals/eval-cot.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/evals/eval-cot.sh -------------------------------------------------------------------------------- /evals/eval-held_in.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/evals/eval-held_in.sh -------------------------------------------------------------------------------- /evals/eval-mmlu.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/evals/eval-mmlu.sh -------------------------------------------------------------------------------- /evals/eval-sglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/evals/eval-sglue.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm100k/finetune_base_code_to_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm100k/finetune_base_code_to_text.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm100k/finetune_large_code_to_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm100k/finetune_large_code_to_text.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm100k/sglue_base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm100k/sglue_base.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm100k/sglue_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm100k/sglue_large.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm100k/t0-train_base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm100k/t0-train_base.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm100k/t0-train_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm100k/t0-train_large.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm_adapt/base_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm_adapt/base_flan.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm_adapt/base_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm_adapt/base_flan2021.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm_adapt/large_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm_adapt/large_flan.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm_adapt/large_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm_adapt/large_flan2021.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm_adapt/xl_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm_adapt/xl_flan.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm_adapt/xl_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm_adapt/xl_flan2021.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm_adapt/xxl_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm_adapt/xxl_flan.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1-lm_adapt/xxl_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1-lm_adapt/xxl_flan2021.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/base_code_to_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/base_code_to_text.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/base_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/base_flan2021.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/base_flan2022.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/base_flan2022.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/base_sglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/base_sglue.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/base_t0-train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/base_t0-train.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/large_code_to_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/large_code_to_text.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/large_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/large_flan2021.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/large_flan2022.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/large_flan2022.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/large_sglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/large_sglue.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/large_t0-train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/large_t0-train.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/xl_code_to_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/xl_code_to_text.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/xl_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/xl_flan2021.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/xl_flan2022.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/xl_flan2022.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/xl_sglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/xl_sglue.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/xxl_code_to_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/xxl_code_to_text.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/xxl_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/xxl_flan2021.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/xxl_flan2022.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/xxl_flan2022.sh -------------------------------------------------------------------------------- /experiments/benchmarks/t5-v1.1/xxl_sglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/benchmarks/t5-v1.1/xxl_sglue.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ablations/v1-1_xl_flan2021_submix.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ablations/v1-1_xl_flan2021_submix.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ablations/v2_xl_flan2021_submix.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ablations/v2_xl_flan2021_submix.sh -------------------------------------------------------------------------------- /experiments/improved_t5/eval_bf16.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/eval_bf16.sh -------------------------------------------------------------------------------- /experiments/improved_t5/eval_fp16.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/eval_fp16.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/finetune_base_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/finetune_base_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/finetune_large_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/finetune_large_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/finetune_xl_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/finetune_xl_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/finetune_xxl_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/finetune_xxl_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/lm_adapt_base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/lm_adapt_base.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/lm_adapt_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/lm_adapt_large.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/lm_adapt_xl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/lm_adapt_xl.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/lm_adapt_xxl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/lm_adapt_xxl.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/s_causal/finetune_base_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/s_causal/finetune_base_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/s_causal/finetune_large_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/s_causal/finetune_large_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/s_causal/finetune_xl_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/s_causal/finetune_xl_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/s_causal/finetune_xxl_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/s_causal/finetune_xxl_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/s_causal/lm_adapt_base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/s_causal/lm_adapt_base.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/s_causal/lm_adapt_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/s_causal/lm_adapt_large.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/s_causal/lm_adapt_xl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/s_causal/lm_adapt_xl.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/s_causal/lm_adapt_xxl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/s_causal/lm_adapt_xxl.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/ul2r/finetune_base_ns_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/ul2r/finetune_base_ns_flan2021.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/ul2r/finetune_xl_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/ul2r/finetune_xl_flan2021.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/ul2r/finetune_xl_flan2022.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/ul2r/finetune_xl_flan2022.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/ul2r/lm_adapt_base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/ul2r/lm_adapt_base.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/ul2r/lm_adapt_base_ns.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/ul2r/lm_adapt_base_ns.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/ul2r/lm_adapt_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/ul2r/lm_adapt_large.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/ul2r/lm_adapt_xl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/ul2r/lm_adapt_xl.sh -------------------------------------------------------------------------------- /experiments/improved_t5/lm_adapt/ul2r/lm_adapt_xxl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/lm_adapt/ul2r/lm_adapt_xxl.sh -------------------------------------------------------------------------------- /experiments/improved_t5/mlm/extend-causal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/mlm/extend-causal.sh -------------------------------------------------------------------------------- /experiments/improved_t5/mlm/extend-prefix.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/mlm/extend-prefix.sh -------------------------------------------------------------------------------- /experiments/improved_t5/mlm/finetune_code_to_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/mlm/finetune_code_to_text.sh -------------------------------------------------------------------------------- /experiments/improved_t5/mlm/finetune_flan2022.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/mlm/finetune_flan2022.sh -------------------------------------------------------------------------------- /experiments/improved_t5/mlm/finetune_sglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/mlm/finetune_sglue.sh -------------------------------------------------------------------------------- /experiments/improved_t5/mlm/finetune_t0.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/mlm/finetune_t0.sh -------------------------------------------------------------------------------- /experiments/improved_t5/mlm/pretrain_mlm.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/mlm/pretrain_mlm.sh -------------------------------------------------------------------------------- /experiments/improved_t5/mlm/pretrain_mlm_causal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/mlm/pretrain_mlm_causal.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_base_code_to_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_base_code_to_text.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_base_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_base_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_base_sglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_base_sglue.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_large_code_to_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_large_code_to_text.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_large_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_large_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_large_sglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_large_sglue.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_xl_code_to_text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_xl_code_to_text.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_xl_flan2021.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_xl_flan2021.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_xl_sglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_xl_sglue.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_xxl_flan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_xxl_flan.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/finetune_xxl_sglue.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/finetune_xxl_sglue.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/pretrain_base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/pretrain_base.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/pretrain_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/pretrain_large.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/pretrain_xl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/pretrain_xl.sh -------------------------------------------------------------------------------- /experiments/improved_t5/ul2_causal/pretrain_xxl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/improved_t5/ul2_causal/pretrain_xxl.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/eval/t0_eval_adamw_post.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/eval/t0_eval_adamw_post.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/eval/t0_eval_adamw_post_rotary.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/eval/t0_eval_adamw_post_rotary.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/eval/t0_eval_adamw_pre.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/eval/t0_eval_adamw_pre.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/eval/t0_eval_alibi_relpos.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/eval/t0_eval_alibi_relpos.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/eval/t0_eval_base_lm100k.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/eval/t0_eval_base_lm100k.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/finetune/t0_train_adafactor_post.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/finetune/t0_train_adafactor_post.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/finetune/t0_train_adafactor_pre.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/finetune/t0_train_adafactor_pre.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/finetune/t0_train_adamw_post.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/finetune/t0_train_adamw_post.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/finetune/t0_train_adamw_post_rotary.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/finetune/t0_train_adamw_post_rotary.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/finetune/t0_train_adamw_pre.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/finetune/t0_train_adamw_pre.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/finetune/t0_train_base_lm100k.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/finetune/t0_train_base_lm100k.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/pretrain/pile_mlm_adafactor_post.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/pretrain/pile_mlm_adafactor_post.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/pretrain/pile_mlm_adafactor_pre.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/pretrain/pile_mlm_adafactor_pre.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/pretrain/pile_mlm_adamw_post.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/pretrain/pile_mlm_adamw_post.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/pretrain/pile_mlm_adamw_post_rotary.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/pretrain/pile_mlm_adamw_post_rotary.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/pretrain/pile_mlm_adamw_pre.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/pretrain/pile_mlm_adamw_pre.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/pretrain/pile_mlm_adamw_pre_rotary.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/pretrain/pile_mlm_adamw_pre_rotary.sh -------------------------------------------------------------------------------- /experiments/preliminary/layernorm/t0_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/layernorm/t0_eval.py -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_128000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_128000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_160000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_160000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_192000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_192000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_224000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_224000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_256000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_256000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_32000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_32000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_64000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_64000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_96000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_10/sglue_finetune_920m_96000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_128000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_128000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_160000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_160000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_192000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_192000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_224000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_224000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_256000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_256000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_32000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_32000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_64000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_64000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_96000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_15/sglue_finetune_920m_96000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_128000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_128000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_160000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_160000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_192000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_192000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_224000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_224000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_256000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_256000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_32000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_32000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_64000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_64000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_96000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_25/sglue_finetune_920m_96000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_128000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_128000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_160000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_160000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_192000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_192000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_224000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_224000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_256000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_256000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_32000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_32000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_64000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_64000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_96000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_50/sglue_finetune_920m_96000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_128000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_128000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_160000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_160000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_192000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_192000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_224000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_224000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_256000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_256000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_32000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_32000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_64000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_64000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_96000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_60/sglue_finetune_920m_96000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_128000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_128000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_160000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_160000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_192000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_192000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_224000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_224000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_256000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_256000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_32000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_32000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_64000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_64000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_96000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/finetune/0_75/sglue_finetune_920m_96000.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_0_10.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_0_10.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_0_15.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_0_15.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_0_25.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_0_25.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_0_50.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_0_50.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_0_75.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_0_75.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_1_00.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_c4/920m_c4_mlm_1_00.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_10.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_10.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_15.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_15.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_25.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_25.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_50.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_50.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_60.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_60.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_75.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/pretrain_pile/920m_pile_mix_0_75.sh -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/sglue_graph.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/sglue_graph.py -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/super_glue_performance_0-10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/super_glue_performance_0-10.png -------------------------------------------------------------------------------- /experiments/preliminary/mixed_pretraining_objectives/super_glue_performance_flop_256k.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/mixed_pretraining_objectives/super_glue_performance_flop_256k.png -------------------------------------------------------------------------------- /experiments/preliminary/partition/pretrain_pile_1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/partition/pretrain_pile_1.sh -------------------------------------------------------------------------------- /experiments/preliminary/partition/pretrain_pile_2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/partition/pretrain_pile_2.sh -------------------------------------------------------------------------------- /experiments/preliminary/partition/pretrain_pile_4.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/partition/pretrain_pile_4.sh -------------------------------------------------------------------------------- /experiments/preliminary/partition/pretrain_pile_8.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/partition/pretrain_pile_8.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1024_1024.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1024_1024.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1024_114.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1024_114.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1024_2048.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1024_2048.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1024_256.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1024_256.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1024_512.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1024_512.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1_1024.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1_1024.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1_114.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1_114.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1_2048.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1_2048.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1_256.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1_256.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1_512.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_1_512.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_512_1024.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_512_1024.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_512_114.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_512_114.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_512_2048.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_512_2048.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_512_256.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_512_256.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_512_512.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/no_alibi_512_512.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1024_1024.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1024_1024.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1024_114.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1024_114.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1024_2048.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1024_2048.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1024_256.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1024_256.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1024_512.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1024_512.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1_1024.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1_1024.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1_114.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1_114.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1_2048.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1_2048.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1_256.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1_256.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1_512.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_1_512.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_512_1024.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_512_1024.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_512_114.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_512_114.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_512_2048.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_512_2048.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_512_256.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_512_256.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_512_512.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/eval_perplexity/with_alibi_512_512.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/finetune_sglue_prefix_lm_no_alibi.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/finetune_sglue_prefix_lm_no_alibi.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/finetune_sglue_prefix_lm_with_alibi.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/finetune_sglue_prefix_lm_with_alibi.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/finetune_sglue_prefix_lm_with_alibi_plus_relpos.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/finetune_sglue_prefix_lm_with_alibi_plus_relpos.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/pretrain_pile_prefix_lm_no_alibi.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/pretrain_pile_prefix_lm_no_alibi.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/pretrain_pile_prefix_lm_with_alibi.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/pretrain_pile_prefix_lm_with_alibi.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/alibi/pretrain_pile_prefix_lm_with_alibi_plus_relpos.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/alibi/pretrain_pile_prefix_lm_with_alibi_plus_relpos.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/pretrain_rotary_pile_mlm.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/pretrain_rotary_pile_mlm.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/eval/t0_eval_alibi_relpos.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/eval/t0_eval_alibi_relpos.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/eval/t0_eval_benchmark.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/eval/t0_eval_benchmark.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/eval/t0_eval_metro_learning_only_training.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/eval/t0_eval_metro_learning_only_training.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/eval/t0_eval_metro_no_alibi.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/eval/t0_eval_metro_no_alibi.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/eval/t0_eval_metro_training.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/eval/t0_eval_metro_training.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/finetune/sglue/sglue_train_benchmark.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/finetune/sglue/sglue_train_benchmark.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/finetune/sglue/sglue_train_rotary.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/finetune/sglue/sglue_train_rotary.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/finetune/sglue/sglue_train_rotary_relpos.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/finetune/sglue/sglue_train_rotary_relpos.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/pretrain_benchmark_pile_mlm.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/pretrain_benchmark_pile_mlm.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/pretrain_rotary_relpos_pile_mlm.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/pretrain_rotary_relpos_pile_mlm.sh -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/sglue_graph.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/sglue_graph.py -------------------------------------------------------------------------------- /experiments/preliminary/positional_embeddings/rotary/t0_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/positional_embeddings/rotary/t0_eval.py -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/110m/sglue_finetune_110m_16000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/110m/sglue_finetune_110m_16000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/110m/sglue_finetune_110m_32000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/110m/sglue_finetune_110m_32000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/110m/sglue_finetune_110m_48000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/110m/sglue_finetune_110m_48000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/110m/sglue_finetune_110m_64000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/110m/sglue_finetune_110m_64000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/110m/sglue_finetune_110m_80000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/110m/sglue_finetune_110m_80000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_128000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_128000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_192000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_192000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_256000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_256000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_320000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_320000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_384000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_384000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_424000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_424000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_448000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_448000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_512000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_512000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_64000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/1_6b/sglue_finetune_1_6b_64000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_16000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_16000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_24000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_24000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_32000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_32000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_40000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_40000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_48000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_48000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_56000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_56000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_64000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_64000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_8000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/25m/sglue_finetune_25m_8000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/920m/pretrain_c4.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/920m/pretrain_c4.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/920m/pretrain_pile.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/920m/pretrain_pile.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_128000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_128000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_160000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_160000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_192000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_192000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_224000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_224000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_256000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_256000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_32000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_32000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_64000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_64000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_96000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/920m/sglue_finetune_920m_96000.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/sglue_finetune.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/sglue_finetune.sh -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/sglue_graph.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/sglue_graph.py -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/sglue_graph_.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/sglue_graph_.py -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/super_glue_performance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/super_glue_performance.png -------------------------------------------------------------------------------- /experiments/preliminary/scaling_laws/super_glue_performance_flop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/experiments/preliminary/scaling_laws/super_glue_performance_flop.png -------------------------------------------------------------------------------- /models/decoder_t5/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/decoder_t5/__init__.py -------------------------------------------------------------------------------- /models/decoder_t5/modeling_decoder_t5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/decoder_t5/modeling_decoder_t5.py -------------------------------------------------------------------------------- /models/scalable_t5/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/README.md -------------------------------------------------------------------------------- /models/scalable_t5/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/__init__.py -------------------------------------------------------------------------------- /models/scalable_t5/alibi_position_biases.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/alibi_position_biases.py -------------------------------------------------------------------------------- /models/scalable_t5/alibi_position_biases_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/alibi_position_biases_test.py -------------------------------------------------------------------------------- /models/scalable_t5/layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/layers.py -------------------------------------------------------------------------------- /models/scalable_t5/layers_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/layers_test.py -------------------------------------------------------------------------------- /models/scalable_t5/local_tiny.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/local_tiny.gin -------------------------------------------------------------------------------- /models/scalable_t5/mt5/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/mt5/__init__.py -------------------------------------------------------------------------------- /models/scalable_t5/mt5/base.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/mt5/base.gin -------------------------------------------------------------------------------- /models/scalable_t5/mt5/large.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/mt5/large.gin -------------------------------------------------------------------------------- /models/scalable_t5/mt5/small.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/mt5/small.gin -------------------------------------------------------------------------------- /models/scalable_t5/mt5/xl.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/mt5/xl.gin -------------------------------------------------------------------------------- /models/scalable_t5/mt5/xxl.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/mt5/xxl.gin -------------------------------------------------------------------------------- /models/scalable_t5/network.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/network.py -------------------------------------------------------------------------------- /models/scalable_t5/network_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/network_test.py -------------------------------------------------------------------------------- /models/scalable_t5/rotary_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/rotary_embedding.py -------------------------------------------------------------------------------- /models/scalable_t5/rotary_embedding_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/rotary_embedding_test.py -------------------------------------------------------------------------------- /models/scalable_t5/t5_1_1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/t5_1_1/__init__.py -------------------------------------------------------------------------------- /models/scalable_t5/t5_1_1/base.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/t5_1_1/base.gin -------------------------------------------------------------------------------- /models/scalable_t5/t5_1_1/examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/t5_1_1/examples/__init__.py -------------------------------------------------------------------------------- /models/scalable_t5/t5_1_1/examples/wmt19_ende_from_scratch.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/t5_1_1/examples/wmt19_ende_from_scratch.gin -------------------------------------------------------------------------------- /models/scalable_t5/t5_1_1/large.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/t5_1_1/large.gin -------------------------------------------------------------------------------- /models/scalable_t5/t5_1_1/small.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/t5_1_1/small.gin -------------------------------------------------------------------------------- /models/scalable_t5/t5_1_1/xl.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/t5_1_1/xl.gin -------------------------------------------------------------------------------- /models/scalable_t5/t5_1_1/xxl.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/models/scalable_t5/t5_1_1/xxl.gin -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/setup.py -------------------------------------------------------------------------------- /tpu-scripts/kill.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/tpu-scripts/kill.sh -------------------------------------------------------------------------------- /tpu-scripts/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/tpu-scripts/run.sh -------------------------------------------------------------------------------- /tpu-scripts/send.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/tpu-scripts/send.sh -------------------------------------------------------------------------------- /tpu-scripts/setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/improved-t5/HEAD/tpu-scripts/setup.sh --------------------------------------------------------------------------------