├── .ci └── FILE_HEADER ├── .github ├── ISSUE_TEMPLATE │ ├── ---bug-report.md │ ├── ---feature-request.md │ └── ---model-questions.md ├── mcp │ └── mcp_pytest.py └── workflows │ ├── code-quality.yaml │ ├── codeql-analysis.yml │ ├── coverage.yaml │ ├── docker.yaml │ ├── pr-cpu.yaml │ ├── pr-gpu.yaml │ ├── pytest-cpu.yaml │ ├── pytest-gpu.yaml │ └── release.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .yamllint.yaml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── TUTORIAL.md ├── assets ├── llm-foundry.png └── radar_blog.png ├── llmfoundry ├── README.md ├── __init__.py ├── callbacks │ ├── __init__.py │ ├── fdiff_callback.py │ ├── generate_callback.py │ ├── model_gauntlet_callback.py │ ├── monolithic_ckpt_callback.py │ ├── resumption_callbacks.py │ └── scheduled_gc_callback.py ├── data │ ├── __init__.py │ ├── data.py │ ├── denoising.py │ ├── finetuning │ │ ├── __init__.py │ │ ├── collator.py │ │ ├── dataloader.py │ │ └── tasks.py │ ├── packing.py │ └── text_data.py ├── models │ ├── __init__.py │ ├── hf │ │ ├── __init__.py │ │ ├── hf_causal_lm.py │ │ ├── hf_fsdp.py │ │ ├── hf_prefix_lm.py │ │ ├── hf_t5.py │ │ └── model_wrapper.py │ ├── layers │ │ ├── __init__.py │ │ ├── attention.py │ │ ├── blocks.py │ │ ├── custom_embedding.py │ │ ├── fc.py │ │ ├── ffn.py │ │ ├── flash_attn_triton.py │ │ ├── llama_attention_monkeypatch.py │ │ └── norm.py │ ├── model_registry.py │ ├── mpt │ │ ├── __init__.py │ │ ├── configuration_mpt.py │ │ └── modeling_mpt.py │ └── utils │ │ ├── __init__.py │ │ ├── adapt_tokenizer.py │ │ ├── hf_prefixlm_converter.py │ │ ├── meta_init_context.py │ │ └── param_init_fns.py ├── optim │ ├── __init__.py │ ├── adaptive_lion.py │ ├── lion.py │ └── outlier_detection.py └── utils │ ├── __init__.py │ ├── builders.py │ ├── config_utils.py │ └── huggingface_hub_utils.py ├── mcli ├── README.md ├── mcli-1b-eval.yaml ├── mcli-1b-max-seq-len-8k.yaml ├── mcli-1b.yaml ├── mcli-benchmark-mpt.yaml ├── mcli-convert-composer-to-hf.yaml ├── mcli-hf-eval.yaml ├── mcli-hf-generate.yaml ├── mcli-llama2-finetune.yaml └── mcli-pretokenize-oci-upload.yaml ├── mosaicml_README.md ├── pyproject.toml ├── scripts ├── data_prep │ ├── README.md │ ├── convert_dataset_hf.py │ ├── convert_dataset_json.py │ ├── convert_finetuning_dataset.py │ └── example_data │ │ ├── .gitignore │ │ └── arxiv.jsonl ├── eval │ ├── README.md │ ├── eval.py │ ├── local_data │ │ ├── .gitignore │ │ ├── MODEL_GAUNTLET.md │ │ ├── commonsense_reasoning │ │ │ ├── bigbench_novel_concepts.jsonl │ │ │ ├── bigbench_strange_stories.jsonl │ │ │ ├── bigbench_strategy_qa.jsonl │ │ │ ├── copa.jsonl │ │ │ ├── openbook_qa.jsonl │ │ │ └── piqa.jsonl │ │ ├── language_understanding │ │ │ ├── bigbench_conceptual_combinations.jsonl │ │ │ ├── bigbench_conlang_translation.jsonl │ │ │ ├── bigbench_language_identification.jsonl │ │ │ ├── hellaswag.jsonl │ │ │ ├── lambada_openai.jsonl │ │ │ ├── winograd_wsc.jsonl │ │ │ └── winogrande.jsonl │ │ ├── reading_comprehension │ │ │ ├── bigbench_understanding_fables.jsonl │ │ │ ├── boolq.jsonl │ │ │ ├── coqa.jsonl │ │ │ ├── narrative_qa.jsonl │ │ │ ├── pubmed_qa_labeled.jsonl │ │ │ └── squad.jsonl │ │ ├── symbolic_problem_solving │ │ │ ├── bigbench_cs_algorithms.jsonl │ │ │ ├── bigbench_dyck_languages.jsonl │ │ │ ├── bigbench_elementary_math_qa.jsonl │ │ │ ├── bigbench_logical_deduction.jsonl │ │ │ ├── bigbench_operators.jsonl │ │ │ ├── bigbench_repeat_copy_logic.jsonl │ │ │ ├── bigbench_simple_arithmetic.jsonl │ │ │ ├── logi_qa.jsonl │ │ │ ├── math_qa.jsonl │ │ │ ├── simple_arithmetic_nospaces.jsonl │ │ │ └── simple_arithmetic_withspaces.jsonl │ │ └── world_knowledge │ │ │ ├── arc_challenge.jsonl │ │ │ ├── arc_easy.jsonl │ │ │ ├── bigbench_misconceptions.jsonl │ │ │ ├── bigbench_movie_recommendation.jsonl │ │ │ ├── bigbench_qa_wikidata.jsonl │ │ │ ├── jeopardy_all.jsonl │ │ │ ├── mmlu.jsonl │ │ │ ├── triviaqa.jsonl │ │ │ └── triviaqa_sm.jsonl │ └── yamls │ │ ├── copa.yaml │ │ ├── hf_eval.yaml │ │ ├── hf_lora_eval.yml │ │ ├── model_gauntlet.yaml │ │ ├── mpt_eval.yaml │ │ ├── tasks.yaml │ │ └── tasks_light.yaml ├── inference │ ├── README.md │ ├── __init__.py │ ├── benchmarking │ │ ├── README.md │ │ ├── assets │ │ │ ├── Latency-for-MPT-30B,-n_input_tok=128.svg │ │ │ ├── Latency-for-MPT-30B,-n_input_tok=2048.svg │ │ │ ├── Latency-for-MPT-7B,-n_input_tok=128.svg │ │ │ ├── Latency-for-MPT-7B,-n_input_tok=2048.svg │ │ │ ├── Latency-vs.-Throughput,-MPT-30B-(n_input_tok=512,-n_output_tok=64).svg │ │ │ ├── Latency-vs.-Throughput,-MPT-7B-(n_input_tok=512,-n_output_tok=64).svg │ │ │ └── Latency-vs.-Throughput-(n_input_tok=512,-n_output_tok=64).svg │ │ ├── benchmark.py │ │ └── yamls │ │ │ ├── 1b.yaml │ │ │ └── 7b.yaml │ ├── convert_composer_to_hf.py │ ├── convert_hf_mpt_to_ft.py │ ├── convert_hf_to_onnx.py │ ├── hf_chat.py │ ├── hf_generate.py │ └── run_mpt_with_ft.py ├── misc │ ├── convert_examples_ckpt.py │ └── update_hub_code.py └── train │ ├── README.md │ ├── benchmarking │ ├── README.md │ ├── collect_results.py │ ├── submit_benchmarks.py │ └── sweep.sh │ ├── finetune_example │ ├── README.md │ ├── gpt2-arc-easy--cpu.yaml │ ├── mpt-7b-arc-easy--gpu.yaml │ ├── preprocessing.py │ └── train.jsonl │ ├── old_train_sparse.py │ ├── scripts │ └── mpt │ │ └── run_sparse_finetune.sh │ ├── train.py │ ├── train_sparse.py │ └── yamls │ ├── finetune │ ├── 1b_local_data_sft.yaml │ ├── 7b_dolly_sft.yaml │ ├── mpt-30b-instruct.yaml │ ├── mpt-7b_dolly_sft.yaml │ ├── mpt-7b_domain_adapt.yaml │ ├── mpt │ │ ├── FT_gsm8k.yaml │ │ ├── FT_gsm8k_noGradClip.yaml │ │ ├── FT_gsm8k_noGradClip_KD.yaml │ │ ├── FT_gsm8k_noGradClip_KDall.yaml │ │ ├── FT_gsm8k_noGradClip_KDall_smallBS.yaml │ │ ├── FT_gsm8k_noGradClip_fp16.yaml │ │ ├── sparse_finetune_with_distillation.yaml │ │ └── test_sparse.yaml │ ├── sparse_finetune_wPacking.yaml │ ├── sparse_finetune_woPacking.yaml │ └── t5-small_dolly_sft.yaml │ └── pretrain │ ├── gpt-neo-125m.yaml │ ├── gpt2-small.yaml │ ├── mpt-125m.yaml │ ├── mpt-13b.yaml │ ├── mpt-1b.yaml │ ├── mpt-30b.yaml │ ├── mpt-350m.yaml │ ├── mpt-3b.yaml │ ├── mpt-70b.yaml │ ├── mpt-760m.yaml │ ├── mpt-7b.yaml │ ├── opt-3b.yaml │ └── testing.yaml ├── setup.py └── tests ├── conftest.py ├── test_data_prep_scripts.py ├── test_dataloader.py ├── test_eval_inputs.py ├── test_flash_triton_torch.py ├── test_hf_config.py ├── test_hf_conversion_script.py ├── test_hf_mpt_gen.py ├── test_hf_v_mpt.py ├── test_icl_datasets.py ├── test_init_fn.py ├── test_llama_patch.py ├── test_model.py ├── test_onnx.py ├── test_tasks.yaml ├── test_tokenizer.py ├── test_train_inputs.py └── test_training.py /.ci/FILE_HEADER: -------------------------------------------------------------------------------- 1 | Copyright 2022 MosaicML LLM Foundry authors 2 | SPDX-License-Identifier: Apache-2.0 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/---bug-report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/ISSUE_TEMPLATE/---bug-report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/---feature-request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/ISSUE_TEMPLATE/---feature-request.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/---model-questions.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/ISSUE_TEMPLATE/---model-questions.md -------------------------------------------------------------------------------- /.github/mcp/mcp_pytest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/mcp/mcp_pytest.py -------------------------------------------------------------------------------- /.github/workflows/code-quality.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/workflows/code-quality.yaml -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/workflows/codeql-analysis.yml -------------------------------------------------------------------------------- /.github/workflows/coverage.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/workflows/coverage.yaml -------------------------------------------------------------------------------- /.github/workflows/docker.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/workflows/docker.yaml -------------------------------------------------------------------------------- /.github/workflows/pr-cpu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/workflows/pr-cpu.yaml -------------------------------------------------------------------------------- /.github/workflows/pr-gpu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/workflows/pr-gpu.yaml -------------------------------------------------------------------------------- /.github/workflows/pytest-cpu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/workflows/pytest-cpu.yaml -------------------------------------------------------------------------------- /.github/workflows/pytest-gpu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/workflows/pytest-gpu.yaml -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.github/workflows/release.yaml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.yamllint.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/.yamllint.yaml -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/README.md -------------------------------------------------------------------------------- /TUTORIAL.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/TUTORIAL.md -------------------------------------------------------------------------------- /assets/llm-foundry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/assets/llm-foundry.png -------------------------------------------------------------------------------- /assets/radar_blog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/assets/radar_blog.png -------------------------------------------------------------------------------- /llmfoundry/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/README.md -------------------------------------------------------------------------------- /llmfoundry/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/__init__.py -------------------------------------------------------------------------------- /llmfoundry/callbacks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/callbacks/__init__.py -------------------------------------------------------------------------------- /llmfoundry/callbacks/fdiff_callback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/callbacks/fdiff_callback.py -------------------------------------------------------------------------------- /llmfoundry/callbacks/generate_callback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/callbacks/generate_callback.py -------------------------------------------------------------------------------- /llmfoundry/callbacks/model_gauntlet_callback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/callbacks/model_gauntlet_callback.py -------------------------------------------------------------------------------- /llmfoundry/callbacks/monolithic_ckpt_callback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/callbacks/monolithic_ckpt_callback.py -------------------------------------------------------------------------------- /llmfoundry/callbacks/resumption_callbacks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/callbacks/resumption_callbacks.py -------------------------------------------------------------------------------- /llmfoundry/callbacks/scheduled_gc_callback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/callbacks/scheduled_gc_callback.py -------------------------------------------------------------------------------- /llmfoundry/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/data/__init__.py -------------------------------------------------------------------------------- /llmfoundry/data/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/data/data.py -------------------------------------------------------------------------------- /llmfoundry/data/denoising.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/data/denoising.py -------------------------------------------------------------------------------- /llmfoundry/data/finetuning/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/data/finetuning/__init__.py -------------------------------------------------------------------------------- /llmfoundry/data/finetuning/collator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/data/finetuning/collator.py -------------------------------------------------------------------------------- /llmfoundry/data/finetuning/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/data/finetuning/dataloader.py -------------------------------------------------------------------------------- /llmfoundry/data/finetuning/tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/data/finetuning/tasks.py -------------------------------------------------------------------------------- /llmfoundry/data/packing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/data/packing.py -------------------------------------------------------------------------------- /llmfoundry/data/text_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/data/text_data.py -------------------------------------------------------------------------------- /llmfoundry/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/__init__.py -------------------------------------------------------------------------------- /llmfoundry/models/hf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/hf/__init__.py -------------------------------------------------------------------------------- /llmfoundry/models/hf/hf_causal_lm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/hf/hf_causal_lm.py -------------------------------------------------------------------------------- /llmfoundry/models/hf/hf_fsdp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/hf/hf_fsdp.py -------------------------------------------------------------------------------- /llmfoundry/models/hf/hf_prefix_lm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/hf/hf_prefix_lm.py -------------------------------------------------------------------------------- /llmfoundry/models/hf/hf_t5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/hf/hf_t5.py -------------------------------------------------------------------------------- /llmfoundry/models/hf/model_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/hf/model_wrapper.py -------------------------------------------------------------------------------- /llmfoundry/models/layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/layers/__init__.py -------------------------------------------------------------------------------- /llmfoundry/models/layers/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/layers/attention.py -------------------------------------------------------------------------------- /llmfoundry/models/layers/blocks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/layers/blocks.py -------------------------------------------------------------------------------- /llmfoundry/models/layers/custom_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/layers/custom_embedding.py -------------------------------------------------------------------------------- /llmfoundry/models/layers/fc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/layers/fc.py -------------------------------------------------------------------------------- /llmfoundry/models/layers/ffn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/layers/ffn.py -------------------------------------------------------------------------------- /llmfoundry/models/layers/flash_attn_triton.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/layers/flash_attn_triton.py -------------------------------------------------------------------------------- /llmfoundry/models/layers/llama_attention_monkeypatch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/layers/llama_attention_monkeypatch.py -------------------------------------------------------------------------------- /llmfoundry/models/layers/norm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/layers/norm.py -------------------------------------------------------------------------------- /llmfoundry/models/model_registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/model_registry.py -------------------------------------------------------------------------------- /llmfoundry/models/mpt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/mpt/__init__.py -------------------------------------------------------------------------------- /llmfoundry/models/mpt/configuration_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/mpt/configuration_mpt.py -------------------------------------------------------------------------------- /llmfoundry/models/mpt/modeling_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/mpt/modeling_mpt.py -------------------------------------------------------------------------------- /llmfoundry/models/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/utils/__init__.py -------------------------------------------------------------------------------- /llmfoundry/models/utils/adapt_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/utils/adapt_tokenizer.py -------------------------------------------------------------------------------- /llmfoundry/models/utils/hf_prefixlm_converter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/utils/hf_prefixlm_converter.py -------------------------------------------------------------------------------- /llmfoundry/models/utils/meta_init_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/utils/meta_init_context.py -------------------------------------------------------------------------------- /llmfoundry/models/utils/param_init_fns.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/models/utils/param_init_fns.py -------------------------------------------------------------------------------- /llmfoundry/optim/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/optim/__init__.py -------------------------------------------------------------------------------- /llmfoundry/optim/adaptive_lion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/optim/adaptive_lion.py -------------------------------------------------------------------------------- /llmfoundry/optim/lion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/optim/lion.py -------------------------------------------------------------------------------- /llmfoundry/optim/outlier_detection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/optim/outlier_detection.py -------------------------------------------------------------------------------- /llmfoundry/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/utils/__init__.py -------------------------------------------------------------------------------- /llmfoundry/utils/builders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/utils/builders.py -------------------------------------------------------------------------------- /llmfoundry/utils/config_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/utils/config_utils.py -------------------------------------------------------------------------------- /llmfoundry/utils/huggingface_hub_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/llmfoundry/utils/huggingface_hub_utils.py -------------------------------------------------------------------------------- /mcli/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mcli/README.md -------------------------------------------------------------------------------- /mcli/mcli-1b-eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mcli/mcli-1b-eval.yaml -------------------------------------------------------------------------------- /mcli/mcli-1b-max-seq-len-8k.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mcli/mcli-1b-max-seq-len-8k.yaml -------------------------------------------------------------------------------- /mcli/mcli-1b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mcli/mcli-1b.yaml -------------------------------------------------------------------------------- /mcli/mcli-benchmark-mpt.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mcli/mcli-benchmark-mpt.yaml -------------------------------------------------------------------------------- /mcli/mcli-convert-composer-to-hf.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mcli/mcli-convert-composer-to-hf.yaml -------------------------------------------------------------------------------- /mcli/mcli-hf-eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mcli/mcli-hf-eval.yaml -------------------------------------------------------------------------------- /mcli/mcli-hf-generate.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mcli/mcli-hf-generate.yaml -------------------------------------------------------------------------------- /mcli/mcli-llama2-finetune.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mcli/mcli-llama2-finetune.yaml -------------------------------------------------------------------------------- /mcli/mcli-pretokenize-oci-upload.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mcli/mcli-pretokenize-oci-upload.yaml -------------------------------------------------------------------------------- /mosaicml_README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/mosaicml_README.md -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/data_prep/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/data_prep/README.md -------------------------------------------------------------------------------- /scripts/data_prep/convert_dataset_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/data_prep/convert_dataset_hf.py -------------------------------------------------------------------------------- /scripts/data_prep/convert_dataset_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/data_prep/convert_dataset_json.py -------------------------------------------------------------------------------- /scripts/data_prep/convert_finetuning_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/data_prep/convert_finetuning_dataset.py -------------------------------------------------------------------------------- /scripts/data_prep/example_data/.gitignore: -------------------------------------------------------------------------------- 1 | !* 2 | -------------------------------------------------------------------------------- /scripts/data_prep/example_data/arxiv.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/data_prep/example_data/arxiv.jsonl -------------------------------------------------------------------------------- /scripts/eval/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/README.md -------------------------------------------------------------------------------- /scripts/eval/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/eval.py -------------------------------------------------------------------------------- /scripts/eval/local_data/.gitignore: -------------------------------------------------------------------------------- 1 | !* 2 | -------------------------------------------------------------------------------- /scripts/eval/local_data/MODEL_GAUNTLET.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/MODEL_GAUNTLET.md -------------------------------------------------------------------------------- /scripts/eval/local_data/commonsense_reasoning/bigbench_novel_concepts.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/commonsense_reasoning/bigbench_novel_concepts.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/commonsense_reasoning/bigbench_strange_stories.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/commonsense_reasoning/bigbench_strange_stories.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/commonsense_reasoning/bigbench_strategy_qa.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/commonsense_reasoning/bigbench_strategy_qa.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/commonsense_reasoning/copa.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/commonsense_reasoning/copa.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/commonsense_reasoning/openbook_qa.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/commonsense_reasoning/openbook_qa.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/commonsense_reasoning/piqa.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/commonsense_reasoning/piqa.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/language_understanding/bigbench_conceptual_combinations.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/language_understanding/bigbench_conceptual_combinations.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/language_understanding/bigbench_conlang_translation.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/language_understanding/bigbench_conlang_translation.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/language_understanding/bigbench_language_identification.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/language_understanding/bigbench_language_identification.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/language_understanding/hellaswag.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/language_understanding/hellaswag.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/language_understanding/lambada_openai.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/language_understanding/lambada_openai.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/language_understanding/winograd_wsc.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/language_understanding/winograd_wsc.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/language_understanding/winogrande.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/language_understanding/winogrande.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/reading_comprehension/bigbench_understanding_fables.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/reading_comprehension/bigbench_understanding_fables.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/reading_comprehension/boolq.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/reading_comprehension/boolq.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/reading_comprehension/coqa.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/reading_comprehension/coqa.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/reading_comprehension/narrative_qa.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/reading_comprehension/narrative_qa.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/reading_comprehension/pubmed_qa_labeled.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/reading_comprehension/pubmed_qa_labeled.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/reading_comprehension/squad.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/reading_comprehension/squad.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/bigbench_cs_algorithms.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/bigbench_cs_algorithms.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/bigbench_dyck_languages.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/bigbench_dyck_languages.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/bigbench_elementary_math_qa.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/bigbench_elementary_math_qa.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/bigbench_logical_deduction.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/bigbench_logical_deduction.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/bigbench_operators.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/bigbench_operators.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/bigbench_repeat_copy_logic.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/bigbench_repeat_copy_logic.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/bigbench_simple_arithmetic.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/bigbench_simple_arithmetic.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/logi_qa.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/logi_qa.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/math_qa.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/math_qa.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/simple_arithmetic_nospaces.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/simple_arithmetic_nospaces.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/symbolic_problem_solving/simple_arithmetic_withspaces.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/symbolic_problem_solving/simple_arithmetic_withspaces.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/world_knowledge/arc_challenge.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/world_knowledge/arc_challenge.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/world_knowledge/arc_easy.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/world_knowledge/arc_easy.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/world_knowledge/bigbench_misconceptions.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/world_knowledge/bigbench_misconceptions.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/world_knowledge/bigbench_movie_recommendation.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/world_knowledge/bigbench_movie_recommendation.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/world_knowledge/bigbench_qa_wikidata.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/world_knowledge/bigbench_qa_wikidata.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/world_knowledge/jeopardy_all.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/world_knowledge/jeopardy_all.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/world_knowledge/mmlu.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/world_knowledge/mmlu.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/world_knowledge/triviaqa.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/world_knowledge/triviaqa.jsonl -------------------------------------------------------------------------------- /scripts/eval/local_data/world_knowledge/triviaqa_sm.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/local_data/world_knowledge/triviaqa_sm.jsonl -------------------------------------------------------------------------------- /scripts/eval/yamls/copa.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/yamls/copa.yaml -------------------------------------------------------------------------------- /scripts/eval/yamls/hf_eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/yamls/hf_eval.yaml -------------------------------------------------------------------------------- /scripts/eval/yamls/hf_lora_eval.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/yamls/hf_lora_eval.yml -------------------------------------------------------------------------------- /scripts/eval/yamls/model_gauntlet.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/yamls/model_gauntlet.yaml -------------------------------------------------------------------------------- /scripts/eval/yamls/mpt_eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/yamls/mpt_eval.yaml -------------------------------------------------------------------------------- /scripts/eval/yamls/tasks.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/yamls/tasks.yaml -------------------------------------------------------------------------------- /scripts/eval/yamls/tasks_light.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/eval/yamls/tasks_light.yaml -------------------------------------------------------------------------------- /scripts/inference/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/README.md -------------------------------------------------------------------------------- /scripts/inference/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 MosaicML LLM Foundry authors 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | __all__ = [] 5 | -------------------------------------------------------------------------------- /scripts/inference/benchmarking/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/README.md -------------------------------------------------------------------------------- /scripts/inference/benchmarking/assets/Latency-for-MPT-30B,-n_input_tok=128.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/assets/Latency-for-MPT-30B,-n_input_tok=128.svg -------------------------------------------------------------------------------- /scripts/inference/benchmarking/assets/Latency-for-MPT-30B,-n_input_tok=2048.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/assets/Latency-for-MPT-30B,-n_input_tok=2048.svg -------------------------------------------------------------------------------- /scripts/inference/benchmarking/assets/Latency-for-MPT-7B,-n_input_tok=128.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/assets/Latency-for-MPT-7B,-n_input_tok=128.svg -------------------------------------------------------------------------------- /scripts/inference/benchmarking/assets/Latency-for-MPT-7B,-n_input_tok=2048.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/assets/Latency-for-MPT-7B,-n_input_tok=2048.svg -------------------------------------------------------------------------------- /scripts/inference/benchmarking/assets/Latency-vs.-Throughput,-MPT-30B-(n_input_tok=512,-n_output_tok=64).svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/assets/Latency-vs.-Throughput,-MPT-30B-(n_input_tok=512,-n_output_tok=64).svg -------------------------------------------------------------------------------- /scripts/inference/benchmarking/assets/Latency-vs.-Throughput,-MPT-7B-(n_input_tok=512,-n_output_tok=64).svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/assets/Latency-vs.-Throughput,-MPT-7B-(n_input_tok=512,-n_output_tok=64).svg -------------------------------------------------------------------------------- /scripts/inference/benchmarking/assets/Latency-vs.-Throughput-(n_input_tok=512,-n_output_tok=64).svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/assets/Latency-vs.-Throughput-(n_input_tok=512,-n_output_tok=64).svg -------------------------------------------------------------------------------- /scripts/inference/benchmarking/benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/benchmark.py -------------------------------------------------------------------------------- /scripts/inference/benchmarking/yamls/1b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/yamls/1b.yaml -------------------------------------------------------------------------------- /scripts/inference/benchmarking/yamls/7b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/benchmarking/yamls/7b.yaml -------------------------------------------------------------------------------- /scripts/inference/convert_composer_to_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/convert_composer_to_hf.py -------------------------------------------------------------------------------- /scripts/inference/convert_hf_mpt_to_ft.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/convert_hf_mpt_to_ft.py -------------------------------------------------------------------------------- /scripts/inference/convert_hf_to_onnx.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/convert_hf_to_onnx.py -------------------------------------------------------------------------------- /scripts/inference/hf_chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/hf_chat.py -------------------------------------------------------------------------------- /scripts/inference/hf_generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/hf_generate.py -------------------------------------------------------------------------------- /scripts/inference/run_mpt_with_ft.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/inference/run_mpt_with_ft.py -------------------------------------------------------------------------------- /scripts/misc/convert_examples_ckpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/misc/convert_examples_ckpt.py -------------------------------------------------------------------------------- /scripts/misc/update_hub_code.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/misc/update_hub_code.py -------------------------------------------------------------------------------- /scripts/train/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/README.md -------------------------------------------------------------------------------- /scripts/train/benchmarking/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/benchmarking/README.md -------------------------------------------------------------------------------- /scripts/train/benchmarking/collect_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/benchmarking/collect_results.py -------------------------------------------------------------------------------- /scripts/train/benchmarking/submit_benchmarks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/benchmarking/submit_benchmarks.py -------------------------------------------------------------------------------- /scripts/train/benchmarking/sweep.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/benchmarking/sweep.sh -------------------------------------------------------------------------------- /scripts/train/finetune_example/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/finetune_example/README.md -------------------------------------------------------------------------------- /scripts/train/finetune_example/gpt2-arc-easy--cpu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/finetune_example/gpt2-arc-easy--cpu.yaml -------------------------------------------------------------------------------- /scripts/train/finetune_example/mpt-7b-arc-easy--gpu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/finetune_example/mpt-7b-arc-easy--gpu.yaml -------------------------------------------------------------------------------- /scripts/train/finetune_example/preprocessing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/finetune_example/preprocessing.py -------------------------------------------------------------------------------- /scripts/train/finetune_example/train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/finetune_example/train.jsonl -------------------------------------------------------------------------------- /scripts/train/old_train_sparse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/old_train_sparse.py -------------------------------------------------------------------------------- /scripts/train/scripts/mpt/run_sparse_finetune.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/scripts/mpt/run_sparse_finetune.sh -------------------------------------------------------------------------------- /scripts/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/train.py -------------------------------------------------------------------------------- /scripts/train/train_sparse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/train_sparse.py -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/1b_local_data_sft.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/1b_local_data_sft.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/7b_dolly_sft.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/7b_dolly_sft.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt-30b-instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt-30b-instruct.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt-7b_dolly_sft.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt-7b_dolly_sft.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt-7b_domain_adapt.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt-7b_domain_adapt.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt/FT_gsm8k.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt/FT_gsm8k.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt/FT_gsm8k_noGradClip.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt/FT_gsm8k_noGradClip.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt/FT_gsm8k_noGradClip_KD.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt/FT_gsm8k_noGradClip_KD.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt/FT_gsm8k_noGradClip_KDall.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt/FT_gsm8k_noGradClip_KDall.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt/FT_gsm8k_noGradClip_KDall_smallBS.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt/FT_gsm8k_noGradClip_KDall_smallBS.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt/FT_gsm8k_noGradClip_fp16.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt/FT_gsm8k_noGradClip_fp16.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt/sparse_finetune_with_distillation.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt/sparse_finetune_with_distillation.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/mpt/test_sparse.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/mpt/test_sparse.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/sparse_finetune_wPacking.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/sparse_finetune_wPacking.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/sparse_finetune_woPacking.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/sparse_finetune_woPacking.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/finetune/t5-small_dolly_sft.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/finetune/t5-small_dolly_sft.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/gpt-neo-125m.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/gpt-neo-125m.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/gpt2-small.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/gpt2-small.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/mpt-125m.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/mpt-125m.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/mpt-13b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/mpt-13b.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/mpt-1b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/mpt-1b.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/mpt-30b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/mpt-30b.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/mpt-350m.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/mpt-350m.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/mpt-3b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/mpt-3b.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/mpt-70b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/mpt-70b.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/mpt-760m.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/mpt-760m.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/mpt-7b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/mpt-7b.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/opt-3b.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/opt-3b.yaml -------------------------------------------------------------------------------- /scripts/train/yamls/pretrain/testing.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/scripts/train/yamls/pretrain/testing.yaml -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/setup.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/test_data_prep_scripts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_data_prep_scripts.py -------------------------------------------------------------------------------- /tests/test_dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_dataloader.py -------------------------------------------------------------------------------- /tests/test_eval_inputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_eval_inputs.py -------------------------------------------------------------------------------- /tests/test_flash_triton_torch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_flash_triton_torch.py -------------------------------------------------------------------------------- /tests/test_hf_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_hf_config.py -------------------------------------------------------------------------------- /tests/test_hf_conversion_script.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_hf_conversion_script.py -------------------------------------------------------------------------------- /tests/test_hf_mpt_gen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_hf_mpt_gen.py -------------------------------------------------------------------------------- /tests/test_hf_v_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_hf_v_mpt.py -------------------------------------------------------------------------------- /tests/test_icl_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_icl_datasets.py -------------------------------------------------------------------------------- /tests/test_init_fn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_init_fn.py -------------------------------------------------------------------------------- /tests/test_llama_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_llama_patch.py -------------------------------------------------------------------------------- /tests/test_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_model.py -------------------------------------------------------------------------------- /tests/test_onnx.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_onnx.py -------------------------------------------------------------------------------- /tests/test_tasks.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_tasks.yaml -------------------------------------------------------------------------------- /tests/test_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_tokenizer.py -------------------------------------------------------------------------------- /tests/test_train_inputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_train_inputs.py -------------------------------------------------------------------------------- /tests/test_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IST-DASLab/SparseFinetuning/HEAD/tests/test_training.py --------------------------------------------------------------------------------