├── LICENSE ├── README.md ├── assets ├── attention_through_layers.png ├── chunking.png ├── full_attention.png ├── kv_padding.png ├── padding.png ├── rolling_cache.png └── sliding_attention.png ├── deploy ├── .dockerignore ├── Dockerfile └── entrypoint.sh ├── finetune ├── __init__.py ├── args.py ├── checkpointing.py ├── data │ ├── __init__.py │ ├── args.py │ ├── build.py │ └── dataset.py ├── distributed.py ├── lora │ ├── __init__.py │ └── linear.py ├── loss.py ├── monitoring │ ├── __init__.py │ ├── metrics_logger.py │ └── utils.py ├── utils.py └── wrapped_model.py ├── main.py ├── mistral ├── __init__.py ├── cache.py ├── model.py ├── rope.py └── tokenizer.py ├── one_file_ref.py ├── pyproject.toml ├── reference ├── 7B_lora.yaml └── params.json ├── requirements.dev.txt ├── requirements.train.txt ├── requirements.txt ├── requirements_hackathon.txt ├── test_generate.py └── train.py /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/README.md -------------------------------------------------------------------------------- /assets/attention_through_layers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/assets/attention_through_layers.png -------------------------------------------------------------------------------- /assets/chunking.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/assets/chunking.png -------------------------------------------------------------------------------- /assets/full_attention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/assets/full_attention.png -------------------------------------------------------------------------------- /assets/kv_padding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/assets/kv_padding.png -------------------------------------------------------------------------------- /assets/padding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/assets/padding.png -------------------------------------------------------------------------------- /assets/rolling_cache.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/assets/rolling_cache.png -------------------------------------------------------------------------------- /assets/sliding_attention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/assets/sliding_attention.png -------------------------------------------------------------------------------- /deploy/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !entrypoint.sh 3 | -------------------------------------------------------------------------------- /deploy/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/deploy/Dockerfile -------------------------------------------------------------------------------- /deploy/entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/deploy/entrypoint.sh -------------------------------------------------------------------------------- /finetune/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /finetune/args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/args.py -------------------------------------------------------------------------------- /finetune/checkpointing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/checkpointing.py -------------------------------------------------------------------------------- /finetune/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /finetune/data/args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/data/args.py -------------------------------------------------------------------------------- /finetune/data/build.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/data/build.py -------------------------------------------------------------------------------- /finetune/data/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/data/dataset.py -------------------------------------------------------------------------------- /finetune/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/distributed.py -------------------------------------------------------------------------------- /finetune/lora/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/lora/__init__.py -------------------------------------------------------------------------------- /finetune/lora/linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/lora/linear.py -------------------------------------------------------------------------------- /finetune/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/loss.py -------------------------------------------------------------------------------- /finetune/monitoring/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /finetune/monitoring/metrics_logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/monitoring/metrics_logger.py -------------------------------------------------------------------------------- /finetune/monitoring/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/monitoring/utils.py -------------------------------------------------------------------------------- /finetune/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/utils.py -------------------------------------------------------------------------------- /finetune/wrapped_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/finetune/wrapped_model.py -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/main.py -------------------------------------------------------------------------------- /mistral/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mistral/cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/mistral/cache.py -------------------------------------------------------------------------------- /mistral/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/mistral/model.py -------------------------------------------------------------------------------- /mistral/rope.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/mistral/rope.py -------------------------------------------------------------------------------- /mistral/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/mistral/tokenizer.py -------------------------------------------------------------------------------- /one_file_ref.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/one_file_ref.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/pyproject.toml -------------------------------------------------------------------------------- /reference/7B_lora.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/reference/7B_lora.yaml -------------------------------------------------------------------------------- /reference/params.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/reference/params.json -------------------------------------------------------------------------------- /requirements.dev.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/requirements.dev.txt -------------------------------------------------------------------------------- /requirements.train.txt: -------------------------------------------------------------------------------- 1 | wandb 2 | tensorboard -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/requirements.txt -------------------------------------------------------------------------------- /requirements_hackathon.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/requirements_hackathon.txt -------------------------------------------------------------------------------- /test_generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/test_generate.py -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistralai-sf24/hackathon/HEAD/train.py --------------------------------------------------------------------------------