├── .gitignore ├── .pre-commit-config.yaml ├── CITATION.cff ├── LICENSE.MD ├── MANIFEST.in ├── README.md ├── cramming ├── __init__.py ├── architectures │ ├── __init__.py │ ├── attention.py │ ├── components.py │ ├── construction.py │ ├── crammed_bert.py │ ├── embeddings.py │ ├── funnel_transformers.py │ ├── huggingface_interface.py │ ├── losses.py │ ├── recurrent_transformers.py │ └── sanity_check.py ├── backend │ ├── __init__.py │ ├── deepspeed_integration.py │ ├── optimizers │ │ ├── __init__.py │ │ ├── adahessian.py │ │ ├── agd.py │ │ ├── nanoT5_optimizer.py │ │ ├── optimizer_modifiers.py │ │ ├── progressive_batching.py │ │ ├── schedulers.py │ │ ├── shampoo │ │ │ ├── CODE_OF_CONDUCT.md │ │ │ ├── CONTRIBUTING.md │ │ │ ├── LICENSE │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── matrix_functions.py │ │ │ ├── shampoo.py │ │ │ └── shampoo_utils.py │ │ └── sophia.py │ ├── prepare_backend.py │ ├── torch_default.py │ └── utils.py ├── config │ ├── __init__.py │ ├── arch │ │ ├── __init__.py │ │ ├── crammed-bert-simple.yaml │ │ ├── crammed-bert.yaml │ │ ├── crammed-large-izsak.yaml │ │ ├── hf-bert-base.yaml │ │ ├── hf-bert-tiny.yaml │ │ └── v1 │ │ │ ├── bert-base.yaml │ │ │ ├── bert-c2.yaml │ │ │ ├── bert-c3.yaml │ │ │ ├── bert-c4.yaml │ │ │ ├── bert-c5.yaml │ │ │ ├── bert-i4.yaml │ │ │ ├── bert-large-izsak.yaml │ │ │ ├── bert-original.yaml │ │ │ ├── bert-tiny.yaml │ │ │ ├── crammed-bamboo.yaml │ │ │ ├── funnel-c2.yaml │ │ │ ├── recurrent-c2.yaml │ │ │ └── sanitycheck.yaml │ ├── cfg_eval.yaml │ ├── cfg_pretrain.yaml │ ├── data │ │ ├── __init__.py │ │ ├── bert-default.yaml │ │ ├── bookcorpus-wikipedia.yaml │ │ ├── c4-subset-processed.yaml │ │ ├── openweb.yaml │ │ ├── oscar.yaml │ │ ├── pile-readymade.yaml │ │ ├── raw-wikipedia-dumps.yaml │ │ ├── roots-mini.yaml │ │ ├── sanity-check-1.yaml │ │ ├── sanity-check-2.yaml │ │ ├── sources │ │ │ ├── ag_news.yaml │ │ │ ├── bookcorpus.yaml │ │ │ ├── c4.yaml │ │ │ ├── dash_books.yaml │ │ │ ├── fake.yaml │ │ │ ├── iwslt.yaml │ │ │ ├── no_code_stackexchange.yaml │ │ │ ├── openwebtext.yaml │ │ │ ├── oscar.yaml │ │ │ ├── raw_wiki_dump.yaml │ │ │ ├── the_pile.yaml │ │ │ ├── the_pileCC.yaml │ │ │ ├── the_pile_dedup.yaml │ │ │ ├── the_pile_natural.yaml │ │ │ ├── the_pile_stream.yaml │ │ │ ├── uncorpus.yaml │ │ │ ├── uspto.yaml │ │ │ ├── wikibooks.yaml │ │ │ ├── wikinews.yaml │ │ │ ├── wikipedia.yaml │ │ │ ├── wikiquote.yaml │ │ │ ├── wikiversity.yaml │ │ │ └── wikivoyage.yaml │ │ ├── the-pile-dedup.yaml │ │ ├── the-pile-natural.yaml │ │ ├── the-pile-stream.yaml │ │ └── the-pile.yaml │ ├── eval │ │ ├── GLUE.yaml │ │ ├── GLUE_sane.yaml │ │ ├── RACE.yaml │ │ ├── SWAG.yaml │ │ ├── __init__.py │ │ ├── mnli.yaml │ │ ├── optim │ │ │ └── adam.yaml │ │ ├── superGLUE.yaml │ │ └── tasks │ │ │ ├── axb.yaml │ │ │ ├── axg.yaml │ │ │ ├── boolq.yaml │ │ │ ├── cb.yaml │ │ │ ├── cola.yaml │ │ │ ├── copa.yaml │ │ │ ├── mnli.yaml │ │ │ ├── mrpc.yaml │ │ │ ├── multirc.yaml │ │ │ ├── qnli.yaml │ │ │ ├── qqp.yaml │ │ │ ├── race.yaml │ │ │ ├── record.yaml │ │ │ ├── rte.yaml │ │ │ ├── sst2.yaml │ │ │ ├── stsb.yaml │ │ │ ├── swag.yaml │ │ │ ├── wic.yaml │ │ │ ├── wnli.yaml │ │ │ └── wsc_fixed.yaml │ ├── hydra │ │ ├── __init__.py │ │ └── job_logging │ │ │ └── custom.yaml │ ├── impl │ │ ├── __init__.py │ │ ├── _default.yaml │ │ ├── deepspeed-hf.yaml │ │ ├── deepspeed.yaml │ │ ├── onnx.yaml │ │ └── torch-default.yaml │ ├── train │ │ ├── __init__.py │ │ ├── bert-base.yaml │ │ ├── bert-izsak.yaml │ │ ├── bert-o4.yaml │ │ ├── bert-original.yaml │ │ ├── optim │ │ │ ├── adafactor.yaml │ │ │ ├── adahessian.yaml │ │ │ ├── adam.yaml │ │ │ ├── adam8bit.yaml │ │ │ ├── adam_classic.yaml │ │ │ ├── adamscale.yaml │ │ │ ├── agd.yaml │ │ │ ├── lion.yaml │ │ │ ├── radam.yaml │ │ │ ├── sgd.yaml │ │ │ ├── shampoo.yaml │ │ │ └── sophia.yaml │ │ ├── optim_mod │ │ │ ├── disabled.yaml │ │ │ ├── larc.yaml │ │ │ ├── lars.yaml │ │ │ ├── progressive.yaml │ │ │ └── sam.yaml │ │ └── v1 │ │ │ ├── bert-o1.yaml │ │ │ ├── bert-o2.yaml │ │ │ └── bert-o3.yaml │ └── wandb │ │ ├── default.yaml │ │ └── none.yaml ├── data │ ├── __init__.py │ ├── curriculum_sorting.py │ ├── deduplicate.py │ ├── downstream_task_preparation.py │ ├── pretraining_preparation.py │ ├── tokenizer_preparation.py │ └── utils.py └── utils.py ├── data_card.md ├── environment.yml ├── eval.py ├── eval_GLUE.sh ├── load_local_model.py ├── model_card.md ├── pretrain.py ├── pyproject.toml ├── sanity_checks2024.sh ├── scripts ├── architecture_ablations_c5_o3.sh ├── data_ablations_a4000.sh ├── data_ablations_a6000.sh ├── eval_baselines.sh ├── final_story_cb_o4_2080ti.sh ├── final_story_cb_o4_a4000.sh ├── final_story_cb_o4_a6000.sh ├── preprocessing.sh ├── reproducing_bert.sh ├── scaling_law_cb_o4_a4000.sh ├── scaling_law_cb_o4_a5000.sh ├── scaling_law_cb_o4_a6000.sh └── training_ablations_c5_o3.sh ├── setup.cfg └── upload_processed_dataset.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/CITATION.cff -------------------------------------------------------------------------------- /LICENSE.MD: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/LICENSE.MD -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/README.md -------------------------------------------------------------------------------- /cramming/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/__init__.py -------------------------------------------------------------------------------- /cramming/architectures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/__init__.py -------------------------------------------------------------------------------- /cramming/architectures/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/attention.py -------------------------------------------------------------------------------- /cramming/architectures/components.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/components.py -------------------------------------------------------------------------------- /cramming/architectures/construction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/construction.py -------------------------------------------------------------------------------- /cramming/architectures/crammed_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/crammed_bert.py -------------------------------------------------------------------------------- /cramming/architectures/embeddings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/embeddings.py -------------------------------------------------------------------------------- /cramming/architectures/funnel_transformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/funnel_transformers.py -------------------------------------------------------------------------------- /cramming/architectures/huggingface_interface.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/huggingface_interface.py -------------------------------------------------------------------------------- /cramming/architectures/losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/losses.py -------------------------------------------------------------------------------- /cramming/architectures/recurrent_transformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/recurrent_transformers.py -------------------------------------------------------------------------------- /cramming/architectures/sanity_check.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/architectures/sanity_check.py -------------------------------------------------------------------------------- /cramming/backend/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/__init__.py -------------------------------------------------------------------------------- /cramming/backend/deepspeed_integration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/deepspeed_integration.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/__init__.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/adahessian.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/adahessian.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/agd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/agd.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/nanoT5_optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/nanoT5_optimizer.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/optimizer_modifiers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/optimizer_modifiers.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/progressive_batching.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/progressive_batching.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/schedulers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/schedulers.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/shampoo/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/shampoo/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /cramming/backend/optimizers/shampoo/CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/shampoo/CONTRIBUTING.md -------------------------------------------------------------------------------- /cramming/backend/optimizers/shampoo/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/shampoo/LICENSE -------------------------------------------------------------------------------- /cramming/backend/optimizers/shampoo/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/shampoo/README.md -------------------------------------------------------------------------------- /cramming/backend/optimizers/shampoo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/shampoo/__init__.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/shampoo/matrix_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/shampoo/matrix_functions.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/shampoo/shampoo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/shampoo/shampoo.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/shampoo/shampoo_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/shampoo/shampoo_utils.py -------------------------------------------------------------------------------- /cramming/backend/optimizers/sophia.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/optimizers/sophia.py -------------------------------------------------------------------------------- /cramming/backend/prepare_backend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/prepare_backend.py -------------------------------------------------------------------------------- /cramming/backend/torch_default.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/torch_default.py -------------------------------------------------------------------------------- /cramming/backend/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/backend/utils.py -------------------------------------------------------------------------------- /cramming/config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cramming/config/arch/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cramming/config/arch/crammed-bert-simple.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/crammed-bert-simple.yaml -------------------------------------------------------------------------------- /cramming/config/arch/crammed-bert.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/crammed-bert.yaml -------------------------------------------------------------------------------- /cramming/config/arch/crammed-large-izsak.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/crammed-large-izsak.yaml -------------------------------------------------------------------------------- /cramming/config/arch/hf-bert-base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/hf-bert-base.yaml -------------------------------------------------------------------------------- /cramming/config/arch/hf-bert-tiny.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/hf-bert-tiny.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/bert-base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/bert-base.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/bert-c2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/bert-c2.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/bert-c3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/bert-c3.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/bert-c4.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/bert-c4.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/bert-c5.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/bert-c5.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/bert-i4.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/bert-i4.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/bert-large-izsak.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/bert-large-izsak.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/bert-original.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/bert-original.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/bert-tiny.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/bert-tiny.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/crammed-bamboo.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/crammed-bamboo.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/funnel-c2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/funnel-c2.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/recurrent-c2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/arch/v1/recurrent-c2.yaml -------------------------------------------------------------------------------- /cramming/config/arch/v1/sanitycheck.yaml: -------------------------------------------------------------------------------- 1 | architectures: 2 | - SanityCheckLM 3 | 4 | width: 1024 # 8352 5 | -------------------------------------------------------------------------------- /cramming/config/cfg_eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/cfg_eval.yaml -------------------------------------------------------------------------------- /cramming/config/cfg_pretrain.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/cfg_pretrain.yaml -------------------------------------------------------------------------------- /cramming/config/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cramming/config/data/bert-default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/bert-default.yaml -------------------------------------------------------------------------------- /cramming/config/data/bookcorpus-wikipedia.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/bookcorpus-wikipedia.yaml -------------------------------------------------------------------------------- /cramming/config/data/c4-subset-processed.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/c4-subset-processed.yaml -------------------------------------------------------------------------------- /cramming/config/data/openweb.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/openweb.yaml -------------------------------------------------------------------------------- /cramming/config/data/oscar.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/oscar.yaml -------------------------------------------------------------------------------- /cramming/config/data/pile-readymade.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/pile-readymade.yaml -------------------------------------------------------------------------------- /cramming/config/data/raw-wikipedia-dumps.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/raw-wikipedia-dumps.yaml -------------------------------------------------------------------------------- /cramming/config/data/roots-mini.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/roots-mini.yaml -------------------------------------------------------------------------------- /cramming/config/data/sanity-check-1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sanity-check-1.yaml -------------------------------------------------------------------------------- /cramming/config/data/sanity-check-2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sanity-check-2.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/ag_news.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/ag_news.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/bookcorpus.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/bookcorpus.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/c4.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/c4.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/dash_books.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/dash_books.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/fake.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/fake.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/iwslt.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/iwslt.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/no_code_stackexchange.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/no_code_stackexchange.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/openwebtext.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/openwebtext.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/oscar.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/oscar.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/raw_wiki_dump.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/raw_wiki_dump.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/the_pile.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/the_pile.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/the_pileCC.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/the_pileCC.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/the_pile_dedup.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/the_pile_dedup.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/the_pile_natural.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/the_pile_natural.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/the_pile_stream.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/the_pile_stream.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/uncorpus.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/uncorpus.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/uspto.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/uspto.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/wikibooks.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/wikibooks.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/wikinews.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/wikinews.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/wikipedia.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/wikipedia.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/wikiquote.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/wikiquote.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/wikiversity.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/wikiversity.yaml -------------------------------------------------------------------------------- /cramming/config/data/sources/wikivoyage.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/sources/wikivoyage.yaml -------------------------------------------------------------------------------- /cramming/config/data/the-pile-dedup.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/the-pile-dedup.yaml -------------------------------------------------------------------------------- /cramming/config/data/the-pile-natural.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/the-pile-natural.yaml -------------------------------------------------------------------------------- /cramming/config/data/the-pile-stream.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/the-pile-stream.yaml -------------------------------------------------------------------------------- /cramming/config/data/the-pile.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/data/the-pile.yaml -------------------------------------------------------------------------------- /cramming/config/eval/GLUE.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/GLUE.yaml -------------------------------------------------------------------------------- /cramming/config/eval/GLUE_sane.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/GLUE_sane.yaml -------------------------------------------------------------------------------- /cramming/config/eval/RACE.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/RACE.yaml -------------------------------------------------------------------------------- /cramming/config/eval/SWAG.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/SWAG.yaml -------------------------------------------------------------------------------- /cramming/config/eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cramming/config/eval/mnli.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/mnli.yaml -------------------------------------------------------------------------------- /cramming/config/eval/optim/adam.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/optim/adam.yaml -------------------------------------------------------------------------------- /cramming/config/eval/superGLUE.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/superGLUE.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/axb.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/axb.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/axg.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/axg.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/boolq.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/boolq.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/cb.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/cb.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/cola.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/cola.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/copa.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/copa.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/mnli.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/mnli.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/mrpc.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/mrpc.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/multirc.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/multirc.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/qnli.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/qnli.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/qqp.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/qqp.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/race.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/race.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/record.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/record.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/rte.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/rte.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/sst2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/sst2.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/stsb.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/stsb.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/swag.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/swag.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/wic.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/wic.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/wnli.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/wnli.yaml -------------------------------------------------------------------------------- /cramming/config/eval/tasks/wsc_fixed.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/eval/tasks/wsc_fixed.yaml -------------------------------------------------------------------------------- /cramming/config/hydra/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cramming/config/hydra/job_logging/custom.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/hydra/job_logging/custom.yaml -------------------------------------------------------------------------------- /cramming/config/impl/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cramming/config/impl/_default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/impl/_default.yaml -------------------------------------------------------------------------------- /cramming/config/impl/deepspeed-hf.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/impl/deepspeed-hf.yaml -------------------------------------------------------------------------------- /cramming/config/impl/deepspeed.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/impl/deepspeed.yaml -------------------------------------------------------------------------------- /cramming/config/impl/onnx.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/impl/onnx.yaml -------------------------------------------------------------------------------- /cramming/config/impl/torch-default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/impl/torch-default.yaml -------------------------------------------------------------------------------- /cramming/config/train/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cramming/config/train/bert-base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/bert-base.yaml -------------------------------------------------------------------------------- /cramming/config/train/bert-izsak.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/bert-izsak.yaml -------------------------------------------------------------------------------- /cramming/config/train/bert-o4.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/bert-o4.yaml -------------------------------------------------------------------------------- /cramming/config/train/bert-original.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/bert-original.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/adafactor.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/adafactor.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/adahessian.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/adahessian.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/adam.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/adam.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/adam8bit.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/adam8bit.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/adam_classic.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/adam_classic.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/adamscale.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/adamscale.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/agd.yaml: -------------------------------------------------------------------------------- 1 | type: AGD 2 | 3 | gain: 1.0 4 | -------------------------------------------------------------------------------- /cramming/config/train/optim/lion.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/lion.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/radam.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/radam.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/sgd.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/sgd.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/shampoo.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/shampoo.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim/sophia.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim/sophia.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim_mod/disabled.yaml: -------------------------------------------------------------------------------- 1 | name: none 2 | -------------------------------------------------------------------------------- /cramming/config/train/optim_mod/larc.yaml: -------------------------------------------------------------------------------- 1 | name: LARC 2 | 3 | trust_coefficient: 0.02 4 | clip: True 5 | eps: 1e-8 6 | -------------------------------------------------------------------------------- /cramming/config/train/optim_mod/lars.yaml: -------------------------------------------------------------------------------- 1 | name: LARS 2 | 3 | trust_coefficient: 0.02 4 | clip: False 5 | eps: 1e-8 6 | -------------------------------------------------------------------------------- /cramming/config/train/optim_mod/progressive.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/optim_mod/progressive.yaml -------------------------------------------------------------------------------- /cramming/config/train/optim_mod/sam.yaml: -------------------------------------------------------------------------------- 1 | name: SAM 2 | rho: 0.05 3 | -------------------------------------------------------------------------------- /cramming/config/train/v1/bert-o1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/v1/bert-o1.yaml -------------------------------------------------------------------------------- /cramming/config/train/v1/bert-o2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/v1/bert-o2.yaml -------------------------------------------------------------------------------- /cramming/config/train/v1/bert-o3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/config/train/v1/bert-o3.yaml -------------------------------------------------------------------------------- /cramming/config/wandb/default.yaml: -------------------------------------------------------------------------------- 1 | enabled: True 2 | entity: YOURNAMEHERE 3 | project: cramming-pretrain 4 | tags: [] 5 | -------------------------------------------------------------------------------- /cramming/config/wandb/none.yaml: -------------------------------------------------------------------------------- 1 | enabled: False 2 | entity: 3 | project: 4 | tags: [] 5 | -------------------------------------------------------------------------------- /cramming/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/data/__init__.py -------------------------------------------------------------------------------- /cramming/data/curriculum_sorting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/data/curriculum_sorting.py -------------------------------------------------------------------------------- /cramming/data/deduplicate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/data/deduplicate.py -------------------------------------------------------------------------------- /cramming/data/downstream_task_preparation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/data/downstream_task_preparation.py -------------------------------------------------------------------------------- /cramming/data/pretraining_preparation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/data/pretraining_preparation.py -------------------------------------------------------------------------------- /cramming/data/tokenizer_preparation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/data/tokenizer_preparation.py -------------------------------------------------------------------------------- /cramming/data/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/data/utils.py -------------------------------------------------------------------------------- /cramming/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/cramming/utils.py -------------------------------------------------------------------------------- /data_card.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/data_card.md -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/environment.yml -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/eval.py -------------------------------------------------------------------------------- /eval_GLUE.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/eval_GLUE.sh -------------------------------------------------------------------------------- /load_local_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/load_local_model.py -------------------------------------------------------------------------------- /model_card.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/model_card.md -------------------------------------------------------------------------------- /pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/pretrain.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/pyproject.toml -------------------------------------------------------------------------------- /sanity_checks2024.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/sanity_checks2024.sh -------------------------------------------------------------------------------- /scripts/architecture_ablations_c5_o3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/architecture_ablations_c5_o3.sh -------------------------------------------------------------------------------- /scripts/data_ablations_a4000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/data_ablations_a4000.sh -------------------------------------------------------------------------------- /scripts/data_ablations_a6000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/data_ablations_a6000.sh -------------------------------------------------------------------------------- /scripts/eval_baselines.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/eval_baselines.sh -------------------------------------------------------------------------------- /scripts/final_story_cb_o4_2080ti.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/final_story_cb_o4_2080ti.sh -------------------------------------------------------------------------------- /scripts/final_story_cb_o4_a4000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/final_story_cb_o4_a4000.sh -------------------------------------------------------------------------------- /scripts/final_story_cb_o4_a6000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/final_story_cb_o4_a6000.sh -------------------------------------------------------------------------------- /scripts/preprocessing.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/preprocessing.sh -------------------------------------------------------------------------------- /scripts/reproducing_bert.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/reproducing_bert.sh -------------------------------------------------------------------------------- /scripts/scaling_law_cb_o4_a4000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/scaling_law_cb_o4_a4000.sh -------------------------------------------------------------------------------- /scripts/scaling_law_cb_o4_a5000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/scaling_law_cb_o4_a5000.sh -------------------------------------------------------------------------------- /scripts/scaling_law_cb_o4_a6000.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/scaling_law_cb_o4_a6000.sh -------------------------------------------------------------------------------- /scripts/training_ablations_c5_o3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/scripts/training_ablations_c5_o3.sh -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/setup.cfg -------------------------------------------------------------------------------- /upload_processed_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonasGeiping/cramming/HEAD/upload_processed_dataset.py --------------------------------------------------------------------------------