├── .dockerignore ├── .env.local ├── .gitattributes ├── .github ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE │ ├── 1-bug-report.yml │ ├── 2-feature-request.yml │ └── config.yml ├── PULL_REQUEST_TEMPLATE.md ├── SECURITY.md └── workflows │ ├── docker.yml │ ├── label_issue.yml │ ├── publish.yml │ └── tests.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CITATION.cff ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── README_zh.md ├── assets ├── logo.png ├── wechat.jpg └── wechat_npu.jpg ├── data ├── README.md ├── README_zh.md ├── alpaca_en_demo.json ├── alpaca_zh_demo.json ├── belle_multiturn │ └── belle_multiturn.py ├── c4_demo.jsonl ├── dataset_info.json ├── dpo_en_demo.json ├── dpo_zh_demo.json ├── glaive_toolcall_en_demo.json ├── glaive_toolcall_zh_demo.json ├── hh_rlhf_en │ └── hh_rlhf_en.py ├── identity.json ├── kto_en_demo.json ├── mllm_audio_demo.json ├── mllm_demo.json ├── mllm_demo_data │ ├── 1.jpg │ ├── 1.mp3 │ ├── 1.mp4 │ ├── 2.avi │ ├── 2.jpg │ ├── 2.wav │ ├── 3.flac │ ├── 3.jpg │ ├── 3.mp4 │ ├── 4.mp3 │ └── 4.mp4 ├── mllm_video_audio_demo.json ├── mllm_video_demo.json ├── ultra_chat │ └── ultra_chat.py └── wiki_demo.txt ├── docker ├── docker-cuda │ ├── Dockerfile │ ├── Dockerfile.base │ └── docker-compose.yml ├── docker-npu │ ├── Dockerfile │ └── docker-compose.yml └── docker-rocm │ ├── Dockerfile │ └── docker-compose.yml ├── evaluation ├── ceval │ ├── ceval.py │ ├── ceval.zip │ └── mapping.json ├── cmmlu │ ├── cmmlu.py │ ├── cmmlu.zip │ └── mapping.json └── mmlu │ ├── mapping.json │ ├── mmlu.py │ └── mmlu.zip ├── examples ├── README.md ├── README_zh.md ├── accelerate │ ├── fsdp_config.yaml │ └── fsdp_config_offload.yaml ├── deepspeed │ ├── ds_z0_config.json │ ├── ds_z2_config.json │ ├── ds_z2_offload_config.json │ ├── ds_z3_config.json │ └── ds_z3_offload_config.json ├── extras │ ├── adam_mini │ │ └── qwen2_full_sft.yaml │ ├── apollo │ │ └── llama3_full_sft.yaml │ ├── badam │ │ └── llama3_full_sft.yaml │ ├── fsdp_qlora │ │ ├── llama3_lora_sft.yaml │ │ └── train.sh │ ├── galore │ │ └── llama3_full_sft.yaml │ ├── llama_pro │ │ ├── expand.sh │ │ └── llama3_freeze_sft.yaml │ ├── loraplus │ │ └── llama3_lora_sft.yaml │ ├── mod │ │ └── llama3_full_sft.yaml │ ├── muon │ │ └── qwen2_full_sft.yaml │ ├── nlg_eval │ │ └── llama3_lora_predict.yaml │ └── pissa │ │ ├── init.sh │ │ └── llama3_lora_sft.yaml ├── inference │ ├── llama3.yaml │ ├── llama3_full_sft.yaml │ ├── llama3_lora_sft.yaml │ └── qwen2_5vl.yaml ├── merge_lora │ ├── llama3_full_sft.yaml │ ├── llama3_gptq.yaml │ ├── llama3_lora_sft.yaml │ └── qwen2_5vl_lora_sft.yaml ├── train_full │ ├── llama3_full_sft.yaml │ └── qwen2_5vl_full_sft.yaml ├── train_lora │ ├── llama3_lora_dpo.yaml │ ├── llama3_lora_eval.yaml │ ├── llama3_lora_kto.yaml │ ├── llama3_lora_ppo.yaml │ ├── llama3_lora_pretrain.yaml │ ├── llama3_lora_reward.yaml │ ├── llama3_lora_sft.sh │ ├── llama3_lora_sft.yaml │ ├── llama3_lora_sft_ds3.yaml │ ├── llama3_lora_sft_ray.yaml │ ├── llama3_preprocess.yaml │ ├── llama4_lora_sft_ds3.yaml │ ├── qwen2_5vl_lora_dpo.yaml │ └── qwen2_5vl_lora_sft.yaml └── train_qlora │ ├── llama3_lora_sft_aqlm.yaml │ ├── llama3_lora_sft_awq.yaml │ ├── llama3_lora_sft_bnb_npu.yaml │ ├── llama3_lora_sft_gptq.yaml │ └── llama3_lora_sft_otfq.yaml ├── pyproject.toml ├── requirements.txt ├── scripts ├── api_example │ ├── test_image.py │ └── test_toolcall.py ├── convert_ckpt │ ├── llamafy_baichuan2.py │ ├── llamafy_qwen.py │ └── tiny_llama4.py ├── eval_bleu_rouge.py ├── llama_pro.py ├── loftq_init.py ├── pissa_init.py ├── qwen_omni_merge.py ├── stat_utils │ ├── cal_flops.py │ ├── cal_lr.py │ ├── cal_mfu.py │ ├── cal_ppl.py │ └── length_cdf.py └── vllm_infer.py ├── setup.py ├── src ├── api.py ├── llamafactory │ ├── __init__.py │ ├── api │ │ ├── __init__.py │ │ ├── app.py │ │ ├── chat.py │ │ ├── common.py │ │ └── protocol.py │ ├── chat │ │ ├── __init__.py │ │ ├── base_engine.py │ │ ├── chat_model.py │ │ ├── hf_engine.py │ │ ├── sglang_engine.py │ │ └── vllm_engine.py │ ├── cli.py │ ├── data │ │ ├── __init__.py │ │ ├── collator.py │ │ ├── converter.py │ │ ├── data_utils.py │ │ ├── formatter.py │ │ ├── loader.py │ │ ├── mm_plugin.py │ │ ├── parser.py │ │ ├── processor │ │ │ ├── __init__.py │ │ │ ├── feedback.py │ │ │ ├── pairwise.py │ │ │ ├── pretrain.py │ │ │ ├── processor_utils.py │ │ │ ├── supervised.py │ │ │ └── unsupervised.py │ │ ├── template.py │ │ └── tool_utils.py │ ├── eval │ │ ├── __init__.py │ │ ├── evaluator.py │ │ └── template.py │ ├── extras │ │ ├── __init__.py │ │ ├── constants.py │ │ ├── env.py │ │ ├── logging.py │ │ ├── misc.py │ │ ├── packages.py │ │ └── ploting.py │ ├── hparams │ │ ├── __init__.py │ │ ├── data_args.py │ │ ├── evaluation_args.py │ │ ├── finetuning_args.py │ │ ├── generating_args.py │ │ ├── model_args.py │ │ ├── parser.py │ │ └── training_args.py │ ├── launcher.py │ ├── model │ │ ├── __init__.py │ │ ├── adapter.py │ │ ├── loader.py │ │ ├── model_utils │ │ │ ├── __init__.py │ │ │ ├── attention.py │ │ │ ├── checkpointing.py │ │ │ ├── embedding.py │ │ │ ├── kv_cache.py │ │ │ ├── liger_kernel.py │ │ │ ├── longlora.py │ │ │ ├── misc.py │ │ │ ├── mod.py │ │ │ ├── moe.py │ │ │ ├── packing.py │ │ │ ├── quantization.py │ │ │ ├── rope.py │ │ │ ├── unsloth.py │ │ │ ├── valuehead.py │ │ │ └── visual.py │ │ └── patcher.py │ ├── third_party │ │ ├── __init__.py │ │ └── muon │ │ │ ├── __init__.py │ │ │ └── muon.py │ ├── train │ │ ├── __init__.py │ │ ├── callbacks.py │ │ ├── dpo │ │ │ ├── __init__.py │ │ │ ├── trainer.py │ │ │ └── workflow.py │ │ ├── kto │ │ │ ├── __init__.py │ │ │ ├── trainer.py │ │ │ └── workflow.py │ │ ├── ppo │ │ │ ├── __init__.py │ │ │ ├── ppo_utils.py │ │ │ ├── trainer.py │ │ │ └── workflow.py │ │ ├── pt │ │ │ ├── __init__.py │ │ │ ├── trainer.py │ │ │ └── workflow.py │ │ ├── rm │ │ │ ├── __init__.py │ │ │ ├── metric.py │ │ │ ├── trainer.py │ │ │ └── workflow.py │ │ ├── sft │ │ │ ├── __init__.py │ │ │ ├── metric.py │ │ │ ├── trainer.py │ │ │ └── workflow.py │ │ ├── test_utils.py │ │ ├── trainer_utils.py │ │ └── tuner.py │ └── webui │ │ ├── __init__.py │ │ ├── chatter.py │ │ ├── common.py │ │ ├── components │ │ ├── __init__.py │ │ ├── chatbot.py │ │ ├── data.py │ │ ├── eval.py │ │ ├── export.py │ │ ├── infer.py │ │ ├── top.py │ │ └── train.py │ │ ├── control.py │ │ ├── css.py │ │ ├── engine.py │ │ ├── interface.py │ │ ├── locales.py │ │ ├── manager.py │ │ └── runner.py ├── train.py └── webui.py └── tests ├── check_license.py ├── data ├── processor │ ├── test_feedback.py │ ├── test_pairwise.py │ ├── test_processor_utils.py │ ├── test_supervised.py │ └── test_unsupervised.py ├── test_collator.py ├── test_converter.py ├── test_formatter.py ├── test_loader.py ├── test_mm_plugin.py └── test_template.py ├── e2e ├── test_chat.py ├── test_sglang.py └── test_train.py ├── eval └── test_eval_template.py ├── model ├── model_utils │ ├── test_add_tokens.py │ ├── test_attention.py │ ├── test_checkpointing.py │ ├── test_misc.py │ ├── test_packing.py │ └── test_visual.py ├── test_base.py ├── test_freeze.py ├── test_full.py ├── test_lora.py └── test_pissa.py ├── train └── test_sft_trainer.py └── version.txt /.dockerignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .git 3 | .github 4 | .venv 5 | cache 6 | docker 7 | saves 8 | hf_cache 9 | ms_cache 10 | om_cache 11 | shared_data 12 | output 13 | .dockerignore 14 | .gitattributes 15 | .gitignore 16 | -------------------------------------------------------------------------------- /.env.local: -------------------------------------------------------------------------------- 1 | # Note: actually we do not support .env, just for reference 2 | # api 3 | API_HOST= 4 | API_PORT= 5 | API_KEY= 6 | API_MODEL_NAME= 7 | API_VERBOSE= 8 | FASTAPI_ROOT_PATH= 9 | MAX_CONCURRENT= 10 | # general 11 | DISABLE_VERSION_CHECK= 12 | FORCE_CHECK_IMPORTS= 13 | ALLOW_EXTRA_ARGS= 14 | LLAMAFACTORY_VERBOSITY= 15 | USE_MODELSCOPE_HUB= 16 | USE_OPENMIND_HUB= 17 | USE_RAY= 18 | RECORD_VRAM= 19 | OPTIM_TORCH= 20 | NPU_JIT_COMPILE= 21 | # torchrun 22 | FORCE_TORCHRUN= 23 | MASTER_ADDR= 24 | MASTER_PORT= 25 | NNODES= 26 | NODE_RANK= 27 | NPROC_PER_NODE= 28 | # wandb 29 | WANDB_DISABLED= 30 | WANDB_PROJECT= 31 | WANDB_API_KEY= 32 | # gradio ui 33 | GRADIO_SHARE= 34 | GRADIO_SERVER_NAME= 35 | GRADIO_SERVER_PORT= 36 | GRADIO_ROOT_PATH= 37 | GRADIO_IPV6= 38 | # setup 39 | ENABLE_SHORT_CONSOLE= 40 | # reserved (do not use) 41 | LLAMABOARD_ENABLED= 42 | LLAMABOARD_WORKDIR= 43 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to LLaMA Factory 2 | 3 | Everyone is welcome to contribute, and we value everybody's contribution. Code contributions are not the only way to help the community. Answering questions, helping others, and improving the documentation are also immensely valuable. 4 | 5 | It also helps us if you spread the word! Reference the library in blog posts about the awesome projects it made possible, shout out on Twitter every time it has helped you, or simply ⭐️ the repository to say thank you. 6 | 7 | However you choose to contribute, please be mindful and respect our [code of conduct](CODE_OF_CONDUCT.md). 8 | 9 | **This guide was heavily inspired by [transformers guide to contributing](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md).** 10 | 11 | ## Ways to contribute 12 | 13 | There are several ways you can contribute to LLaMA Factory: 14 | 15 | * Fix outstanding issues with the existing code. 16 | * Submit issues related to bugs or desired new features. 17 | * Contribute to the examples or to the documentation. 18 | 19 | ### Style guide 20 | 21 | LLaMA Factory follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html), check it for details. 22 | 23 | ### Create a Pull Request 24 | 25 | 1. Fork the [repository](https://github.com/hiyouga/LLaMA-Factory) by clicking on the [Fork](https://github.com/hiyouga/LLaMA-Factory/fork) button on the repository's page. This creates a copy of the code under your GitHub user account. 26 | 27 | 2. Clone your fork to your local disk, and add the base repository as a remote: 28 | 29 | ```bash 30 | git clone git@github.com:[username]/LLaMA-Factory.git 31 | cd LLaMA-Factory 32 | git remote add upstream https://github.com/hiyouga/LLaMA-Factory.git 33 | ``` 34 | 35 | 3. Create a new branch to hold your development changes: 36 | 37 | ```bash 38 | git checkout -b dev_your_branch 39 | ``` 40 | 41 | 4. Set up a development environment by running the following command in a virtual environment: 42 | 43 | ```bash 44 | pip install -e ".[dev]" 45 | ``` 46 | 47 | If LLaMA Factory was already installed in the virtual environment, remove it with `pip uninstall llamafactory` before reinstalling it in editable mode with the -e flag. 48 | 49 | 5. Check code before commit: 50 | 51 | ```bash 52 | make commit 53 | make style && make quality 54 | make test 55 | ``` 56 | 57 | 6. Submit changes: 58 | 59 | ```bash 60 | git add . 61 | git commit -m "commit message" 62 | git fetch upstream 63 | git rebase upstream/main 64 | git push -u origin dev_your_branch 65 | ``` 66 | 67 | 7. Create a merge request from your branch `dev_your_branch` at [origin repo](https://github.com/hiyouga/LLaMA-Factory). 68 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/1-bug-report.yml: -------------------------------------------------------------------------------- 1 | name: "\U0001F41B Bug / help" 2 | description: Create a report to help us improve the LLaMA Factory 3 | labels: ["bug", "pending"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Issues included in **[FAQs](https://github.com/hiyouga/LLaMA-Factory/issues/4614)** or those with **insufficient** information may be closed without a response. 9 | 已经包含在 **[常见问题](https://github.com/hiyouga/LLaMA-Factory/issues/4614)** 内或提供信息**不完整**的 issues 可能不会被回复。 10 | 11 | - type: markdown 12 | attributes: 13 | value: | 14 | Please do not create issues that are not related to framework bugs under this category, use **[Discussions](https://github.com/hiyouga/LLaMA-Factory/discussions/categories/q-a)** instead. 15 | 请勿在此分类下创建和框架 bug 无关的 issues，训练问题求助请使用 **[讨论区](https://github.com/hiyouga/LLaMA-Factory/discussions/categories/q-a)**。 16 | 17 | - type: checkboxes 18 | id: reminder 19 | attributes: 20 | label: Reminder 21 | description: | 22 | Please ensure you have read the above rules carefully and searched the existing issues (including FAQs). 23 | 请确保您已经认真阅读了上述规则并且搜索过现有的 issues（包括常见问题）。 24 | 25 | options: 26 | - label: I have read the above rules and searched the existing issues. 27 | required: true 28 | 29 | - type: textarea 30 | id: system-info 31 | validations: 32 | required: true 33 | attributes: 34 | label: System Info 35 | description: | 36 | Please share your system info with us. You can run the command **llamafactory-cli env** and copy-paste its output below. 37 | 请提供您的系统信息。您可以在命令行运行 **llamafactory-cli env** 并将其输出复制到该文本框中。 38 | 39 | placeholder: llamafactory version, platform, python version, ... 40 | 41 | - type: textarea 42 | id: reproduction 43 | validations: 44 | required: true 45 | attributes: 46 | label: Reproduction 47 | description: | 48 | Please provide entry arguments, error messages and stack traces that reproduces the problem. 49 | 请提供入口参数，错误日志以及异常堆栈以便于我们复现问题。 50 | 51 | value: | 52 | ```text 53 | Put your message here. 54 | ``` 55 | 56 | - type: textarea 57 | id: others 58 | validations: 59 | required: false 60 | attributes: 61 | label: Others 62 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2-feature-request.yml: -------------------------------------------------------------------------------- 1 | name: "\U0001F680 Feature request" 2 | description: Submit a request for a new feature 3 | labels: ["enhancement", "pending"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Please do not create issues that are not related to new features under this category. 9 | 请勿在此分类下创建和新特性无关的 issues。 10 | 11 | - type: checkboxes 12 | id: reminder 13 | attributes: 14 | label: Reminder 15 | description: | 16 | Please ensure you have read the above rules carefully and searched the existing issues. 17 | 请确保您已经认真阅读了上述规则并且搜索过现有的 issues。 18 | 19 | options: 20 | - label: I have read the above rules and searched the existing issues. 21 | required: true 22 | 23 | - type: textarea 24 | id: description 25 | validations: 26 | required: true 27 | attributes: 28 | label: Description 29 | description: | 30 | A clear and concise description of the feature proposal. 31 | 请详细描述您希望加入的新功能特性。 32 | 33 | - type: textarea 34 | id: contribution 35 | validations: 36 | required: false 37 | attributes: 38 | label: Pull Request 39 | description: | 40 | Have you already created the relevant PR and submitted the code? 41 | 您是否已经创建了相关 PR 并提交了代码？ 42 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # What does this PR do? 2 | 3 | Fixes # (issue) 4 | 5 | ## Before submitting 6 | 7 | - [ ] Did you read the [contributor guideline](https://github.com/hiyouga/LLaMA-Factory/blob/main/.github/CONTRIBUTING.md)? 8 | - [ ] Did you write any new necessary tests? 9 | -------------------------------------------------------------------------------- /.github/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting Security Issues 2 | 3 | To report a security issue, please use the GitHub Security Advisory ["Report a Vulnerability"](https://github.com/hiyouga/LLaMA-Factory/security/advisories/new) tab. 4 | 5 | We will send a response indicating the next steps in handling your report. After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance. 6 | 7 | Report security bugs in third-party modules to the person or team maintaining the module. 8 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: docker 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - "main" 8 | paths: 9 | - "**/*.py" 10 | - "requirements.txt" 11 | - "docker/**" 12 | - ".github/workflows/*.yml" 13 | pull_request: 14 | branches: 15 | - "main" 16 | paths: 17 | - "**/*.py" 18 | - "requirements.txt" 19 | - "docker/**" 20 | - ".github/workflows/*.yml" 21 | 22 | jobs: 23 | build: 24 | runs-on: ubuntu-latest 25 | 26 | concurrency: 27 | group: ${{ github.workflow }}-${{ github.ref }} 28 | cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} 29 | 30 | environment: 31 | name: docker 32 | url: https://hub.docker.com/r/hiyouga/llamafactory 33 | 34 | steps: 35 | - name: Free up disk space 36 | run: | 37 | df -h 38 | sudo rm -rf /usr/share/dotnet 39 | sudo rm -rf /opt/ghc 40 | sudo rm -rf /opt/hostedtoolcache 41 | df -h 42 | 43 | - name: Checkout 44 | uses: actions/checkout@v4 45 | 46 | - name: Set up Docker Buildx 47 | uses: docker/setup-buildx-action@v3 48 | 49 | - name: Login to Docker Hub 50 | if: github.event_name != 'pull_request' 51 | uses: docker/login-action@v3 52 | with: 53 | username: ${{ vars.DOCKERHUB_USERNAME }} 54 | password: ${{ secrets.DOCKERHUB_TOKEN }} 55 | 56 | - name: Build and push Docker image 57 | uses: docker/build-push-action@v6 58 | with: 59 | context: . 60 | file: ./docker/docker-cuda/Dockerfile 61 | build-args: | 62 | EXTRAS=metrics,deepspeed,liger-kernel 63 | push: ${{ github.event_name != 'pull_request' }} 64 | tags: docker.io/hiyouga/llamafactory:latest 65 | cache-from: type=gha 66 | cache-to: type=gha,mode=max 67 | -------------------------------------------------------------------------------- /.github/workflows/label_issue.yml: -------------------------------------------------------------------------------- 1 | name: label_issue 2 | 3 | on: 4 | issues: 5 | types: 6 | - opened 7 | 8 | jobs: 9 | label_issue: 10 | runs-on: ubuntu-latest 11 | 12 | permissions: 13 | issues: write 14 | 15 | steps: 16 | - env: 17 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 18 | ISSUE_URL: ${{ github.event.issue.html_url }} 19 | ISSUE_TITLE: ${{ github.event.issue.title }} 20 | run: | 21 | LABEL="" 22 | NPU_KEYWORDS=(npu huawei ascend 华为 昇腾) 23 | ISSUE_TITLE_LOWER=$(echo $ISSUE_TITLE | tr '[:upper:]' '[:lower:]') 24 | for KEYWORD in ${NPU_KEYWORDS[@]}; do 25 | if [[ $ISSUE_TITLE_LOWER == *$KEYWORD* ]] && [[ $ISSUE_TITLE_LOWER != *input* ]]; then 26 | LABEL="npu" 27 | break 28 | fi 29 | done 30 | if [ -n "$LABEL" ]; then 31 | gh issue edit $ISSUE_URL --add-label $LABEL 32 | fi 33 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: publish 2 | 3 | on: 4 | workflow_dispatch: 5 | release: 6 | types: 7 | - published 8 | 9 | jobs: 10 | publish: 11 | name: Upload release to PyPI 12 | 13 | runs-on: ubuntu-latest 14 | 15 | environment: 16 | name: release 17 | url: https://pypi.org/p/llamafactory 18 | 19 | permissions: 20 | id-token: write 21 | 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | 26 | - name: Set up Python 27 | uses: actions/setup-python@v5 28 | with: 29 | python-version: "3.9" 30 | 31 | - name: Build package 32 | run: | 33 | make build 34 | 35 | - name: Publish package 36 | uses: pypa/gh-action-pypi-publish@release/v1 37 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - "main" 8 | paths: 9 | - "**.py" 10 | - "requirements.txt" 11 | - ".github/workflows/*.yml" 12 | pull_request: 13 | branches: 14 | - "main" 15 | paths: 16 | - "**.py" 17 | - "requirements.txt" 18 | - ".github/workflows/*.yml" 19 | 20 | jobs: 21 | tests: 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | python: 26 | - "3.9" 27 | - "3.10" 28 | - "3.11" 29 | - "3.12" 30 | os: 31 | - "ubuntu-latest" 32 | - "windows-latest" 33 | - "macos-13" 34 | transformers: 35 | - null 36 | include: # test backward compatibility 37 | - python: "3.9" 38 | os: "ubuntu-latest" 39 | transformers: "4.45.0" 40 | - python: "3.9" 41 | os: "ubuntu-latest" 42 | transformers: "4.49.0" 43 | - python: "3.9" 44 | os: "ubuntu-latest" 45 | transformers: "4.51.0" 46 | 47 | runs-on: ${{ matrix.os }} 48 | 49 | concurrency: 50 | group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}-${{ matrix.python }}-${{ matrix.transformers }} 51 | cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} 52 | 53 | env: 54 | HF_TOKEN: ${{ secrets.HF_TOKEN }} 55 | OS_NAME: ${{ matrix.os }} 56 | 57 | steps: 58 | - name: Checkout 59 | uses: actions/checkout@v4 60 | 61 | - name: Set up Python 62 | uses: actions/setup-python@v5 63 | with: 64 | python-version: ${{ matrix.python }} 65 | cache: "pip" 66 | cache-dependency-path: "**/requirements*.txt" 67 | 68 | - name: Install dependencies 69 | run: | 70 | python -m pip install --upgrade pip 71 | python -m pip install ".[torch,dev]" 72 | 73 | - name: Install transformers 74 | if: ${{ matrix.transformers }} 75 | run: | 76 | python -m pip install "transformers==${{ matrix.transformers }}" 77 | 78 | - name: Cache files 79 | id: hf-hub-cache 80 | uses: actions/cache@v4 81 | with: 82 | path: ${{ runner.temp }}/huggingface 83 | key: huggingface-${{ matrix.os }}-${{ matrix.python }}-${{ matrix.transformers }}-${{ hashFiles('tests/version.txt') }} 84 | 85 | - name: Check quality 86 | run: | 87 | make style && make quality 88 | 89 | - name: Check license 90 | run: | 91 | make license 92 | 93 | - name: Check build 94 | run: | 95 | make build 96 | 97 | - name: Test with pytest 98 | run: | 99 | make test 100 | env: 101 | HF_HOME: ${{ runner.temp }}/huggingface 102 | HF_HUB_OFFLINE: "${{ steps.hf-hub-cache.outputs.cache-hit == 'true' && '1' || '0' }}" 103 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: check-ast 6 | - id: check-added-large-files 7 | args: ['--maxkb=25000'] 8 | - id: check-merge-conflict 9 | - id: check-yaml 10 | - id: debug-statements 11 | - id: end-of-file-fixer 12 | - id: trailing-whitespace 13 | args: [--markdown-linebreak-ext=md] 14 | - id: no-commit-to-branch 15 | args: ['--branch', 'main'] 16 | 17 | - repo: https://github.com/asottile/pyupgrade 18 | rev: v3.17.0 19 | hooks: 20 | - id: pyupgrade 21 | args: [--py38-plus] 22 | 23 | - repo: https://github.com/astral-sh/ruff-pre-commit 24 | rev: v0.6.9 25 | hooks: 26 | - id: ruff 27 | args: [--fix] 28 | - id: ruff-format 29 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | date-released: 2024-03 3 | message: "If you use this software, please cite it as below." 4 | authors: 5 | - family-names: "Zheng" 6 | given-names: "Yaowei" 7 | - family-names: "Zhang" 8 | given-names: "Richong" 9 | - family-names: "Zhang" 10 | given-names: "Junhao" 11 | - family-names: "Ye" 12 | given-names: "Yanhan" 13 | - family-names: "Luo" 14 | given-names: "Zheyan" 15 | - family-names: "Feng" 16 | given-names: "Zhangchi" 17 | - family-names: "Ma" 18 | given-names: "Yongqiang" 19 | title: "LlamaFactory: Unified Efficient Fine-Tuning of 100+ Language Models" 20 | url: "https://arxiv.org/abs/2403.13372" 21 | preferred-citation: 22 | type: conference-paper 23 | conference: 24 | name: "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 3: System Demonstrations)" 25 | authors: 26 | - family-names: "Zheng" 27 | given-names: "Yaowei" 28 | - family-names: "Zhang" 29 | given-names: "Richong" 30 | - family-names: "Zhang" 31 | given-names: "Junhao" 32 | - family-names: "Ye" 33 | given-names: "Yanhan" 34 | - family-names: "Luo" 35 | given-names: "Zheyan" 36 | - family-names: "Feng" 37 | given-names: "Zhangchi" 38 | - family-names: "Ma" 39 | given-names: "Yongqiang" 40 | title: "LlamaFactory: Unified Efficient Fine-Tuning of 100+ Language Models" 41 | url: "https://arxiv.org/abs/2403.13372" 42 | year: 2024 43 | publisher: "Association for Computational Linguistics" 44 | address: "Bangkok, Thailand" 45 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE requirements.txt 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build commit license quality style test 2 | 3 | check_dirs := scripts src tests setup.py 4 | 5 | build: 6 | pip3 install build && python3 -m build 7 | 8 | commit: 9 | pre-commit install 10 | pre-commit run --all-files 11 | 12 | license: 13 | python3 tests/check_license.py $(check_dirs) 14 | 15 | quality: 16 | ruff check $(check_dirs) 17 | ruff format --check $(check_dirs) 18 | 19 | style: 20 | ruff check $(check_dirs) --fix 21 | ruff format $(check_dirs) 22 | 23 | test: 24 | CUDA_VISIBLE_DEVICES= WANDB_DISABLED=true pytest -vv tests/ 25 | -------------------------------------------------------------------------------- /assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hiyouga/LLaMA-Factory/65aa86ed3978151eb01bd5ef2c8df20e178a7d30/assets/logo.png -------------------------------------------------------------------------------- /assets/wechat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hiyouga/LLaMA-Factory/65aa86ed3978151eb01bd5ef2c8df20e178a7d30/assets/wechat.jpg -------------------------------------------------------------------------------- /assets/wechat_npu.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hiyouga/LLaMA-Factory/65aa86ed3978151eb01bd5ef2c8df20e178a7d30/assets/wechat_npu.jpg -------------------------------------------------------------------------------- /data/mllm_audio_demo.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "messages": [ 4 | { 5 | "content": "