├── .github └── workflows │ ├── codeql.yml │ ├── netlify-docs-build.yml │ ├── pytest-server-test.yml │ ├── pytest.yml │ ├── release.yml │ └── ruff.yml ├── .gitignore ├── .python-version ├── .vscode ├── launch.json └── settings.json ├── LICENSE ├── README.md ├── api.py ├── constraints.txt ├── docker ├── README.md ├── cpu │ ├── Dockerfile.cpu │ ├── README.md │ ├── deploy.ps1 │ ├── deploy.sh │ └── docker-compose.yml.tpl └── gpu │ └── nvidia │ ├── Dockerfile.cuda │ ├── README.md │ ├── deploy.ps1 │ ├── deploy.sh │ └── docker-compose.yml.tpl ├── install.sh ├── pytest.ini ├── requirements-gpu.in ├── requirements-no-gpu-uv.txt ├── requirements-rocm-uv.txt ├── requirements-uv.txt ├── requirements.in ├── ruff.toml ├── run.sh ├── scripts ├── aws │ ├── aws.sh │ └── awsservers.py ├── azure │ ├── README.md │ └── terraform │ │ ├── cloud-init │ │ └── cloud-init.yaml │ │ ├── main.tf │ │ ├── modules │ │ ├── compute │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf │ │ ├── network │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf │ │ └── resource_group │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf │ │ ├── outputs.tf │ │ ├── run.sh │ │ ├── terraform.tfvars.tpl │ │ └── variables.tf ├── check_sizes_of_models.py ├── conda_activate.sh └── xml-rpc-client-example │ ├── example-with-retries.py │ ├── example.py │ ├── tlab_sdk_client │ ├── __init__.py │ ├── callbacks │ │ ├── __init__.py │ │ └── hf_callback.py │ └── client.py │ ├── train_example.py │ └── training_client_example.py ├── test ├── API.http ├── README.md ├── __init__.py ├── api │ ├── __init__.py │ ├── conftest.py │ ├── test_api.py │ ├── test_config.py │ ├── test_data.py │ ├── test_evals.py │ ├── test_export.py │ ├── test_jobs.py │ ├── test_model.py │ ├── test_plugins.py │ ├── test_prompts.py │ ├── test_recipes.py │ ├── test_recipes_new.py │ ├── test_server_info.py │ ├── test_tasks.py │ ├── test_tools.py │ ├── test_train.py │ ├── test_user.py │ └── test_workflows.py ├── db │ ├── __init__.py │ └── test_db.py ├── installer │ ├── __init__.py │ └── test_install_dot_sh.py ├── plugins │ ├── __init__.py │ ├── plugin.schema.json │ └── test_plugins_schema.py ├── server │ ├── __init__.py │ ├── conftest.py │ ├── test_config.py │ └── test_server_info.py ├── shared │ └── test_shared.py └── tmp │ └── .gitignore └── transformerlab ├── __init__.py ├── db.py ├── fastchat_openai_api.py ├── galleries ├── dataset-gallery.json ├── exp-recipe-gallery.json ├── model-gallery.json ├── model-group-gallery.json ├── plugin-gallery.json ├── prompt-gallery.json └── recipe-gallery.json ├── launch_header_text.txt ├── models ├── __init__.py ├── basemodel.py ├── filesystemmodel.py ├── huggingfacemodel.py ├── localmodel.py ├── model_helper.py ├── modelstore.py └── ollamamodel.py ├── plugin_sdk ├── __init__.py ├── plugin_harness.py └── transformerlab │ ├── plugin.py │ └── sdk │ ├── __init__.py │ └── v1 │ ├── __init__.py │ ├── evals.py │ ├── export.py │ ├── generate.py │ ├── tlab_plugin.py │ └── train.py ├── plugins ├── airllm_mlx_server │ ├── index.json │ ├── main.py │ └── setup.sh ├── autotrain_sft_trainer │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── basic_evals │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── batched_generation_datasets │ ├── index.json │ ├── info.md │ ├── main.py │ ├── requests_batching.py │ └── setup.sh ├── common-eleuther-ai-lm-eval-harness-mlx │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── common-eleuther-ai-lm-eval-harness │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── deepeval_llm_judge │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── deepeval_objective │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── dpo_orpo_simpo_trainer_llama_factory │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── eleuther-ai-lm-evaluation-harness-mlx │ ├── index.json │ ├── info.md │ ├── main.py │ ├── setup.sh │ ├── tasklist.txt │ └── tasklist_rough.txt ├── eleuther-ai-lm-evaluation-harness │ ├── index.json │ ├── info.md │ ├── main.py │ ├── setup.sh │ └── tasklist.txt ├── embedding_model_trainer │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── fastchat_server │ ├── index.json │ ├── main.py │ ├── model_worker.py │ └── setup.sh ├── fastchat_vision_server │ ├── index.json │ ├── main.py │ └── setup.sh ├── generate_rag_outputs │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── gguf_exporter │ ├── index.json │ ├── main.py │ └── setup.sh ├── grpo_trainer_multi_gpu │ ├── index.json │ ├── main.py │ └── setup.sh ├── inference_evals │ ├── index.json │ ├── main.py │ ├── requests_batching.py │ └── setup.sh ├── llama_cpp_server │ ├── index.json │ ├── main.py │ └── setup.sh ├── llama_trainer │ ├── index.json │ ├── main.py │ └── setup.sh ├── llama_trainer_multi_gpu │ ├── index.json │ ├── main.py │ └── setup.sh ├── llamafile_exporter │ ├── index.json │ ├── main.py │ └── setup.sh ├── llamaindex_simple_document_search │ ├── index.json │ ├── main.py │ └── setup.sh ├── mlx_exporter │ ├── index.json │ ├── main.py │ └── setup.sh ├── mlx_lora_trainer │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── mlx_rlaif_trainer │ ├── index.json │ ├── main.py │ ├── mlx-rlhf │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── README.md │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── data_utils.py │ │ │ ├── digit_seq_rewards.py │ │ │ ├── generate_digit_data.py │ │ │ ├── imessage_chat_data.py │ │ │ ├── increasing_mult_1_test.jsonl │ │ │ ├── increasing_mult_1_train.jsonl │ │ │ ├── increasing_mult_1_valid.jsonl │ │ │ ├── increasing_mult_2_test.jsonl │ │ │ ├── increasing_mult_2_train.jsonl │ │ │ ├── increasing_mult_2_valid.jsonl │ │ │ ├── reward_function_increasing_mult_2_train.jsonl │ │ │ └── reward_function_increasing_mult_2_valid.jsonl │ │ ├── imessage_bot.md │ │ ├── mlx_ppo_trainer.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── config.py │ │ │ ├── convert.py │ │ │ ├── fuse.py │ │ │ ├── llama.py │ │ │ ├── lora.py │ │ │ ├── mixtral.py │ │ │ └── prompt_tuning.py │ │ ├── ppo_training.py │ │ ├── pytorch_baseline │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── pytorch_ppo_trainer.py │ │ │ ├── pytorch_ppo_training.py │ │ │ ├── pytorch_sft.py │ │ │ ├── pytorch_talk_to_model.py │ │ │ └── utils.py │ │ ├── requirements.txt │ │ ├── sequential_digits.md │ │ ├── sft.py │ │ ├── talk_to_model.py │ │ └── utils.py │ └── setup.sh ├── mlx_server │ ├── index.json │ ├── main.py │ └── setup.sh ├── mlx_vlm_server │ ├── generate.py │ ├── index.json │ ├── language.py │ ├── llava.py │ ├── main.py │ ├── setup.sh │ └── vision.py ├── nanotron_pretrainer │ ├── convert_nanotron_to_hf.py │ ├── convert_weights.py │ ├── index.json │ ├── main.py │ └── setup.sh ├── ollama_server │ ├── index.json │ ├── main.py │ └── setup.sh ├── red_teaming │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── reward_modeling_llama_factory │ ├── dpo_en_demo.json │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── sample_plugin │ ├── index.json │ ├── main.py │ └── setup.sh ├── sft_llama_factory │ ├── index.json │ ├── main.py │ └── setup.sh ├── synthesizer_docs │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── synthesizer_raw_text │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── synthesizer_scratch │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── synthetic_dataset_kit │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── synthetic_dataset_rag │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── t5_lora_trainer │ ├── index.json │ ├── main.py │ └── setup.sh ├── unsloth_grpo_trainer │ ├── index.json │ ├── main.py │ └── setup.sh ├── vllm_server │ ├── index.json │ ├── main.py │ └── setup.sh ├── vlm_trainer │ ├── index.json │ ├── main.py │ ├── qwen_vl_utils.py │ └── setup.sh └── yourbench_data_gen │ ├── index.json │ ├── info.md │ ├── main.py │ └── setup.sh ├── routers ├── __init__.py ├── batched_prompts.py ├── config.py ├── data.py ├── evals.py ├── experiment │ ├── conversations.py │ ├── documents.py │ ├── evals.py │ ├── experiment.py │ ├── export.py │ ├── generations.py │ ├── plugins.py │ └── rag.py ├── job_sdk.py ├── jobs.py ├── model.py ├── plugins.py ├── prompts.py ├── recipes.py ├── serverinfo.py ├── tasks.py ├── tools.py ├── train.py ├── users.py └── workflows.py ├── schemas └── user.py ├── services └── user_service.py ├── shared ├── __init__.py ├── batched_requests.py ├── dirs.py ├── download_huggingface_model.py ├── galleries.py ├── models │ ├── __init__.py │ └── models.py └── shared.py └── tools ├── calculator └── main.py └── weather └── main.py /.github/workflows/netlify-docs-build.yml: -------------------------------------------------------------------------------- 1 | # This workflow triggers a Netlify docs build when a new GitHub release is published 2 | name: Trigger Netlify Docs Build on Release 3 | 4 | on: 5 | release: 6 | types: [published] 7 | 8 | jobs: 9 | build-docs: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Trigger Netlify Build Hook 13 | env: 14 | NETLIFY_BUILD_HOOK: ${{ secrets.NETLIFY_BUILD_HOOK }} 15 | run: | 16 | if [ -z "$NETLIFY_BUILD_HOOK" ]; then 17 | echo "NETLIFY_BUILD_HOOK secret is not set!" 18 | exit 1 19 | fi 20 | curl -X POST -d '{}' "$NETLIFY_BUILD_HOOK" 21 | -------------------------------------------------------------------------------- /.github/workflows/pytest-server-test.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies and run tests 2 | 3 | name: Test for Install and Serving 4 | 5 | on: 6 | push: 7 | branches: ["main"] 8 | pull_request: 9 | branches: ["main"] 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | python-version: ["3.11"] 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v3 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | pip install uv 28 | ENV_NAME="transformerlab" 29 | TLAB_DIR="$HOME/.transformerlab" 30 | TLAB_CODE_DIR="${TLAB_DIR}/src" 31 | TLAB_STATIC_WEB_DIR="${TLAB_DIR}/webapp" 32 | mkdir -p "${TLAB_DIR}" 33 | mkdir -p "${TLAB_CODE_DIR}" 34 | mkdir -p "${TLAB_STATIC_WEB_DIR}" 35 | ./install.sh install_conda 36 | ./install.sh create_conda_environment 37 | ./install.sh install_dependencies 38 | uv pip install --system pytest pytest-asyncio jsonschema requests 39 | - name: Now run the server by calling ./run.sh and wait for it to be ready 40 | run: | 41 | ./run.sh & 42 | sleep 20 # Wait for the server to start 43 | - name: Test the API Server and DB Setup 44 | run: | 45 | pytest -m live_server test/server/ 46 | -------------------------------------------------------------------------------- /.github/workflows/pytest.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies and run tests 2 | 3 | name: Pytest 4 | 5 | on: 6 | push: 7 | branches: ["main"] 8 | pull_request: 9 | branches: ["main"] 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | python-version: ["3.11"] 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v3 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | pip install uv 28 | uv pip install --system pytest pytest-asyncio jsonschema requests pytest-cov shellcheck-py 29 | uv pip install --system -r requirements-no-gpu-uv.txt --upgrade --index=https://download.pytorch.org/whl/cpu 30 | - name: Test with pytest 31 | run: | 32 | pytest --cov=transformerlab --cov-branch --cov-report=xml 33 | 34 | - name: Upload results to Codecov 35 | uses: codecov/codecov-action@v5 36 | with: 37 | token: ${{ secrets.CODECOV_TOKEN }} 38 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Build and Push Docker Images 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | build-and-push: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - name: Checkout code 13 | uses: actions/checkout@v2 14 | 15 | - name: Set up Docker Buildx 16 | uses: docker/setup-buildx-action@v1 17 | 18 | - name: Log in to Docker Hub 19 | uses: docker/login-action@v1 20 | with: 21 | username: ${{ secrets.DOCKERHUB_USERNAME }} 22 | password: ${{ secrets.DOCKERHUB_TOKEN }} 23 | 24 | - name: Extract version from tag 25 | id: get_version 26 | run: | 27 | # Remove the 'v' prefix if it exists, e.g., v1.2.3 -> 1.2.3 28 | echo "VERSION=${GITHUB_REF#refs/tags/}" | sed 's/^v//' >> $GITHUB_ENV 29 | 30 | - name: Build CPU image 31 | run: docker build -f docker/cpu/Dockerfile.cpu -t transformerlab/api:${VERSION} . 32 | 33 | - name: Build GPU image 34 | run: docker build -f docker/gpu/nvidia/Dockerfile.cuda -t transformerlab/api:${VERSION}-cuda . 35 | 36 | - name: Push CPU image 37 | run: docker push transformerlab/api:${VERSION} 38 | 39 | - name: Push GPU image 40 | run: docker push transformerlab/api:${VERSION}-cuda 41 | -------------------------------------------------------------------------------- /.github/workflows/ruff.yml: -------------------------------------------------------------------------------- 1 | name: Ruff 2 | on: [ push, pull_request ] 3 | jobs: 4 | ruff: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v4 8 | - uses: astral-sh/ruff-action@v3 9 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.11 2 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python Debugger: FastAPI", 9 | "type": "debugpy", 10 | "request": "launch", 11 | "module": "uvicorn", 12 | "args": [ 13 | "api:app", 14 | "--reload", 15 | "--port", 16 | "8338", 17 | "--host", 18 | "0.0.0.0" 19 | ], 20 | "jinja": true 21 | } 22 | ] 23 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[python]": { 3 | "editor.defaultFormatter": "charliermarsh.ruff", 4 | "editor.formatOnSave": true 5 | }, 6 | "[json]": { 7 | "editor.defaultFormatter": "esbenp.prettier-vscode", 8 | "editor.formatOnSave": true 9 | }, 10 | "python.testing.pytestArgs": ["test"], 11 | "python.testing.unittestEnabled": false, 12 | "python.testing.pytestEnabled": true 13 | } 14 | -------------------------------------------------------------------------------- /constraints.txt: -------------------------------------------------------------------------------- 1 | torch==2.6.0 2 | -------------------------------------------------------------------------------- /docker/cpu/Dockerfile.cpu: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim 2 | 3 | # Install minimal dependencies required for the install script 4 | RUN apt-get update && apt-get install -y \ 5 | curl \ 6 | git 7 | 8 | # Download and run the install.sh script from GitHub. 9 | RUN curl -fsSL https://raw.githubusercontent.com/transformerlab/transformerlab-api/refs/heads/main/install.sh | bash 10 | 11 | EXPOSE 8338 12 | 13 | VOLUME ["/root/.transformerlab/"] 14 | 15 | WORKDIR /root/.transformerlab/src/ 16 | 17 | RUN chmod +x ./run.sh 18 | 19 | # The entrypoint is set to run the Transformer Lab launcher script. 20 | ENTRYPOINT ["/root/.transformerlab/src/run.sh"] -------------------------------------------------------------------------------- /docker/cpu/deploy.ps1: -------------------------------------------------------------------------------- 1 | # Check if curl exists, install if missing 2 | if (-not (Get-Command curl -ErrorAction SilentlyContinue)) { 3 | Write-Output "curl not found, installing curl..." 4 | winget install --id curl.curl -e --source winget 5 | } 6 | 7 | # GitHub repo details 8 | $repo = "transformerlab/transformerlab-api" 9 | $url = "https://api.github.com/repos/$repo/releases/latest" 10 | 11 | # Get latest version from GitHub 12 | $response = Invoke-RestMethod -Uri $url -Method Get 13 | if (-not $response.tag_name) { 14 | Write-Output "Failed to fetch the latest version." 15 | exit 1 16 | } 17 | 18 | # Remove leading 'v' 19 | $version = $response.tag_name.TrimStart("v") 20 | 21 | Write-Output "Latest TransformerLab API version: $version" 22 | 23 | # Set environment paths 24 | $homeDir = $env:USERPROFILE.Replace("\", "/") 25 | 26 | # Read template and substitute version and HOME variable 27 | $template = Get-Content -Path ".\docker-compose.yml.tpl" -Raw 28 | $content = $template ` 29 | -replace '\$\{VERSION\}', $version ` 30 | -replace '\$\{HOME\}', $homeDir 31 | 32 | # Write final docker-compose file 33 | $content | Out-File -FilePath ".\docker-compose.yml" -Encoding utf8 34 | 35 | Write-Output "Generated docker-compose.yml with image version: $version and HOME: $homeDir" 36 | 37 | # Deploy using docker compose 38 | docker compose up -d 39 | -------------------------------------------------------------------------------- /docker/cpu/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Check if curl is installed; install it if it's not. 4 | if ! command -v curl &> /dev/null; then 5 | echo "curl not found. Installing curl..." 6 | sudo apt-get update && sudo apt-get install -y curl 7 | fi 8 | 9 | # Fetch latest version from GitHub 10 | REPO="transformerlab/transformerlab-api" 11 | RAW_VERSION=$(curl -s "https://api.github.com/repos/$REPO/releases/latest" \ 12 | | grep -o '"tag_name": *"[^"]*' \ 13 | | sed 's/"tag_name": *"//') 14 | 15 | if [ -z "$RAW_VERSION" ]; then 16 | echo "Failed to fetch the latest version." 17 | exit 1 18 | fi 19 | 20 | # Remove leading 'v' from version tag if present 21 | VERSION="${RAW_VERSION#v}" 22 | 23 | echo "Latest TransformerLab API version: $VERSION" 24 | 25 | # Export the version for envsubst 26 | export VERSION 27 | 28 | # Generate docker-compose.yml dynamically 29 | envsubst < docker-compose.yml.tpl > docker-compose.yml 30 | 31 | echo "Generated docker-compose.yml with image version: ${VERSION}" 32 | 33 | # Deploy container 34 | docker compose up -d 35 | 36 | -------------------------------------------------------------------------------- /docker/cpu/docker-compose.yml.tpl: -------------------------------------------------------------------------------- 1 | services: 2 | transformerlab-api: 3 | image: transformerlab/api:${VERSION} 4 | container_name: transformerlab-api 5 | ports: 6 | - "8338:8338" 7 | volumes: 8 | - transformerlab_data_cpu:/root/.transformerlab/ 9 | - ${HOME}/.cache:/root/.cache 10 | - ${HOME}/.transformerlab/workspace:/root/.transformerlab/workspace 11 | restart: unless-stopped 12 | tty: true 13 | stdin_open: true 14 | 15 | volumes: 16 | transformerlab_data_cpu: -------------------------------------------------------------------------------- /docker/gpu/nvidia/Dockerfile.cuda: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:12.1.1-devel-ubuntu22.04 2 | 3 | SHELL ["/bin/bash", "--login", "-c"] 4 | 5 | # Set noninteractive mode 6 | ENV DEBIAN_FRONTEND=noninteractive 7 | 8 | # Install minimal dependencies required for the install script 9 | RUN apt-get update && apt-get install -y \ 10 | curl \ 11 | git \ 12 | python3.11 \ 13 | python3-pip \ 14 | python-is-python3 15 | 16 | # Download and run the install.sh script from GitHub. 17 | RUN curl -fsSL https://raw.githubusercontent.com/transformerlab/transformerlab-api/refs/heads/main/install.sh | bash 18 | 19 | EXPOSE 8338 20 | 21 | VOLUME ["/root/.transformerlab/"] 22 | 23 | WORKDIR /root/.transformerlab/src/ 24 | 25 | RUN chmod +x ./run.sh 26 | 27 | # The entrypoint is set to run the Transformer Lab launcher script. 28 | ENTRYPOINT ["/root/.transformerlab/src/run.sh"] 29 | -------------------------------------------------------------------------------- /docker/gpu/nvidia/deploy.ps1: -------------------------------------------------------------------------------- 1 | # Check if curl exists, install if missing 2 | if (-not (Get-Command curl -ErrorAction SilentlyContinue)) { 3 | Write-Output "curl not found, installing curl..." 4 | winget install --id curl.curl -e --source winget 5 | } 6 | 7 | # GitHub repo details 8 | $repo = "transformerlab/transformerlab-api" 9 | $url = "https://api.github.com/repos/$repo/releases/latest" 10 | 11 | # Get latest version from GitHub 12 | $response = Invoke-RestMethod -Uri $url -Method Get 13 | if (-not $response.tag_name) { 14 | Write-Output "Failed to fetch the latest version." 15 | exit 1 16 | } 17 | 18 | # Remove leading 'v' 19 | $version = $response.tag_name.TrimStart("v") 20 | 21 | Write-Output "Latest TransformerLab API version: $version" 22 | 23 | # Set environment paths 24 | $homeDir = $env:USERPROFILE.Replace("\", "/") 25 | 26 | # Read template and substitute version and HOME variable 27 | $template = Get-Content -Path ".\docker-compose.yml.tpl" -Raw 28 | $content = $template ` 29 | -replace '\$\{VERSION\}', $version ` 30 | -replace '\$\{HOME\}', $homeDir 31 | 32 | # Write final docker-compose file 33 | $content | Out-File -FilePath ".\docker-compose.yml" -Encoding utf8 34 | 35 | Write-Output "Generated docker-compose.yml with image version: $version-cuda and HOME: $homeDir" 36 | 37 | # Deploy using docker compose 38 | docker compose up -d 39 | -------------------------------------------------------------------------------- /docker/gpu/nvidia/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Check if curl is installed; install it if it's not. 4 | if ! command -v curl &> /dev/null; then 5 | echo "curl not found. Installing curl..." 6 | sudo apt-get update && sudo apt-get install -y curl 7 | fi 8 | 9 | # Fetch latest version from GitHub 10 | REPO="transformerlab/transformerlab-api" 11 | RAW_VERSION=$(curl -s "https://api.github.com/repos/$REPO/releases/latest" \ 12 | | grep -o '"tag_name": *"[^"]*' \ 13 | | sed 's/"tag_name": *"//') 14 | 15 | if [ -z "$RAW_VERSION" ]; then 16 | echo "Failed to fetch the latest version." 17 | exit 1 18 | fi 19 | 20 | # Remove leading 'v' from version tag if present 21 | VERSION="${RAW_VERSION#v}" 22 | 23 | echo "Latest TransformerLab API version: $VERSION" 24 | 25 | # Export the version for envsubst 26 | export VERSION 27 | 28 | # Generate docker-compose.yml dynamically 29 | envsubst < docker-compose.yml.tpl > docker-compose.yml 30 | 31 | echo "Generated docker-compose.yml with image version: ${VERSION}-cuda" 32 | 33 | # Deploy container 34 | docker compose up -d 35 | 36 | -------------------------------------------------------------------------------- /docker/gpu/nvidia/docker-compose.yml.tpl: -------------------------------------------------------------------------------- 1 | services: 2 | transformerlab-api: 3 | image: transformerlab/api:${VERSION}-cuda 4 | container_name: transformerlab-api 5 | ports: 6 | - "8338:8338" 7 | ipc: host 8 | volumes: 9 | - transformerlab_data:/root/.transformerlab/ 10 | - ${HOME}/.cache:/root/.cache 11 | - ${HOME}/.transformerlab/workspace:/root/.transformerlab/workspace 12 | deploy: 13 | resources: 14 | reservations: 15 | devices: 16 | - driver: nvidia 17 | count: "all" 18 | capabilities: [gpu] 19 | restart: unless-stopped 20 | tty: true 21 | stdin_open: true 22 | 23 | volumes: 24 | transformerlab_data: -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # pytest.ini 2 | [pytest] 3 | asyncio_mode = auto 4 | asyncio_default_fixture_loop_scope = module 5 | testpaths = 6 | test 7 | markers = 8 | live_server: mark test as requiring a live server 9 | addopts = -m "not live_server" 10 | # to run the live_server tests explicitly, use: 11 | # pytest -m live_server -------------------------------------------------------------------------------- /requirements-gpu.in: -------------------------------------------------------------------------------- 1 | bitsandbytes 2 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | accelerate 2 | aiofiles 3 | aiosqlite 4 | datasets 5 | einops 6 | evaluate 7 | fastapi 8 | fastapi-users[sqlalchemy] 9 | nvidia-ml-py 10 | packaging 11 | psutil 12 | python-multipart 13 | pydantic>= 2.0 14 | nltk==3.9.1 15 | scipy 16 | sentencepiece 17 | tensorboard 18 | tiktoken 19 | torch==2.7.0 20 | torchaudio==2.7.0 21 | torchvision==0.22.0 22 | transformers==4.51.3 23 | peft 24 | watchfiles 25 | wandb==0.19.10 26 | tensorboardX 27 | werkzeug 28 | sqlalchemy[asyncio] 29 | sentence-transformers 30 | markitdown[all] 31 | hf_xet 32 | macmon-python 33 | mcp[cli] 34 | transformerlab-inference 35 | pyrsmi -------------------------------------------------------------------------------- /ruff.toml: -------------------------------------------------------------------------------- 1 | # Exclude a variety of commonly ignored directories. 2 | exclude = [ 3 | ".bzr", 4 | ".direnv", 5 | ".eggs", 6 | ".git", 7 | ".git-rewrite", 8 | ".hg", 9 | ".ipynb_checkpoints", 10 | ".mypy_cache", 11 | ".nox", 12 | ".pants.d", 13 | ".pyenv", 14 | ".pytest_cache", 15 | ".pytype", 16 | ".ruff_cache", 17 | ".svn", 18 | ".tox", 19 | ".venv", 20 | ".vscode", 21 | "__pypackages__", 22 | "_build", 23 | "buck-out", 24 | "build", 25 | "dist", 26 | "node_modules", 27 | "site-packages", 28 | "venv", 29 | ] 30 | 31 | # Same as Black. 32 | line-length = 120 33 | indent-width = 4 34 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | ENV_NAME="transformerlab" 5 | TLAB_DIR="$HOME/.transformerlab" 6 | TLAB_CODE_DIR="${TLAB_DIR}/src" 7 | 8 | MINIFORGE_ROOT=${TLAB_DIR}/miniforge3 9 | CONDA_BIN=${MINIFORGE_ROOT}/bin/conda 10 | ENV_DIR=${TLAB_DIR}/envs/${ENV_NAME} 11 | CUSTOM_ENV=false 12 | 13 | TLABHOST="0.0.0.0" 14 | PORT="8338" 15 | 16 | RELOAD=false 17 | 18 | # echo "Your shell is $SHELL" 19 | # echo "Conda's binary is at ${CONDA_BIN}" 20 | # echo "Your current directory is $(pwd)" 21 | 22 | err_report() { 23 | echo "Error in run.sh on line $1" 24 | } 25 | 26 | # trap 'err_report $LINENO' ERR 27 | 28 | if ! command -v ${CONDA_BIN} &> /dev/null; then 29 | echo "❌ Conda is not installed at ${MINIFORGE_ROOT}. Please run ./install.sh and try again." 30 | else 31 | echo "✅ Conda is installed." 32 | fi 33 | 34 | while getopts crp:h: flag 35 | do 36 | case "${flag}" in 37 | c) CUSTOM_ENV=true;; 38 | r) RELOAD=true;; 39 | p) PORT=${OPTARG};; 40 | h) TLABHOST=${OPTARG};; 41 | esac 42 | done 43 | 44 | # Print out everything that was discovered above 45 | # echo "👏 Using host: ${HOST} 46 | # 👏 Using port: ${PORT} 47 | # 👏 Using reload: ${RELOAD} 48 | # 👏 Using custom environment: ${CUSTOM_ENV}" 49 | 50 | if [ "$CUSTOM_ENV" = true ]; then 51 | echo "🔧 Using current conda environment, I won't activate for you" 52 | else 53 | # echo "👏 Using default conda environment: ${ENV_DIR}" 54 | echo "👏 Enabling conda in shell" 55 | 56 | eval "$(${CONDA_BIN} shell.bash hook)" 57 | 58 | echo "👏 Activating transformerlab conda environment" 59 | conda activate "${ENV_DIR}" 60 | fi 61 | 62 | # Check if the uvicorn command works: 63 | if ! command -v uvicorn &> /dev/null; then 64 | echo "❌ Uvicorn is not installed. This usually means that the installation of dependencies failed. Run ./install.sh to install the dependencies." 65 | exit 1 66 | else 67 | echo -n "" 68 | # echo "✅ Uvicorn is installed." 69 | fi 70 | 71 | # Check if NVIDIA GPU is available and add necessary paths 72 | if command -v nvidia-smi &> /dev/null; then 73 | echo "✅ NVIDIA GPU detected, adding CUDA libraries to path" 74 | # Add common NVIDIA library paths 75 | export LD_LIBRARY_PATH=${ENV_DIR}/lib:$LD_LIBRARY_PATH 76 | elif command -v rocminfo &> /dev/null; then 77 | echo "✅ AMD GPU detected, adding appropriate libraries to path" 78 | export PATH=$PATH:/opt/rocm/bin:/opt/rocm/rocprofiler/bin:/opt/rocm/opencl/bin 79 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib:/opt/rocm/lib64 80 | fi 81 | 82 | echo "▶️ Starting the API server:" 83 | if [ "$RELOAD" = true ]; then 84 | echo "🔁 Reload the server on file changes" 85 | uv run -v uvicorn api:app --reload --port ${PORT} --host ${TLABHOST} 86 | else 87 | uv run -v uvicorn api:app --port ${PORT} --host ${TLABHOST} --no-access-log 88 | fi 89 | -------------------------------------------------------------------------------- /scripts/aws/awsservers.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | 4 | 5 | def list_servers(): 6 | servers = json.load(sys.stdin)["Reservations"][0]["Instances"] 7 | 8 | print("Server Type Status Public IP Server Name") 9 | print( 10 | "---------------------+--------------------+-------------+-----------------+------------------------------------------" 11 | ) 12 | for server in servers: 13 | id = server["InstanceId"] 14 | instance_type = server["InstanceType"] 15 | status = server.get("State", {}).get("Name", "") 16 | ip_address = "" 17 | dns_name = "" 18 | 19 | if status == "running": 20 | ip_address = server.get("PublicIpAddress", "") 21 | dns_name = server.get("PublicDnsName", "") 22 | 23 | print(f"{id} {instance_type:20} {status:13} {ip_address:17} {dns_name}") 24 | 25 | 26 | # Take first parameter and use it to call a function 27 | if __name__ == "__main__": 28 | # args: [0] = current file, [1] = function name, [2:] = function args : (*unpacked) 29 | args = sys.argv 30 | globals()[args[1]](*args[2:]) 31 | -------------------------------------------------------------------------------- /scripts/azure/terraform/cloud-init/cloud-init.yaml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | package_update: true 3 | package_upgrade: true 4 | packages: 5 | - git 6 | - tar 7 | - curl 8 | - wget 9 | 10 | bootcmd: 11 | - mkdir -p /home/azureuser/.transformerlab 12 | 13 | write_files: 14 | - path: /home/azureuser/start_transformerlab.sh 15 | owner: root:root 16 | permissions: '0755' 17 | content: | 18 | #!/bin/bash 19 | set -e 20 | 21 | export HOME=/home/azureuser 22 | 23 | echo "Checking if Transformer Lab is installed..." 24 | if [ ! -f /home/azureuser/.transformerlab/install.sh ]; then 25 | 26 | echo "Downloading install.sh into /home/azureuser/.transformerlab..." 27 | wget -O /home/azureuser/.transformerlab/install.sh https://raw.githubusercontent.com/transformerlab/transformerlab-api/refs/heads/main/install.sh 28 | chmod +x /home/azureuser/.transformerlab/install.sh 29 | 30 | echo "Running install.sh..." 31 | bash /home/azureuser/.transformerlab/install.sh 32 | 33 | echo "Chowning everything under /home/azureuser to azureuser..." 34 | chown -R azureuser:azureuser /home/azureuser 35 | 36 | fi 37 | 38 | echo "Transformer Lab installed." 39 | 40 | echo "Running run.sh..." 41 | cd /home/azureuser/.transformerlab/src 42 | bash run.sh 43 | 44 | - path: /etc/systemd/system/transformerlab.service 45 | owner: root:root 46 | permissions: '0644' 47 | content: | 48 | [Unit] 49 | Description=Transformer Lab API Service 50 | After=network.target 51 | 52 | [Service] 53 | Type=simple 54 | ExecStart=/home/azureuser/start_transformerlab.sh 55 | Restart=always 56 | RestartSec=10 57 | 58 | [Install] 59 | WantedBy=multi-user.target 60 | 61 | runcmd: 62 | - systemctl daemon-reload 63 | - systemctl enable transformerlab.service 64 | - systemctl start transformerlab.service 65 | -------------------------------------------------------------------------------- /scripts/azure/terraform/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | azurerm = { 4 | source = "hashicorp/azurerm" 5 | version = ">= 3.0.0" 6 | } 7 | tls = { 8 | source = "hashicorp/tls" 9 | version = ">= 4.0.0" 10 | } 11 | } 12 | } 13 | 14 | # Automatically fetch the subscription id from your current Azure CLI login. 15 | data "azurerm_client_config" "current" {} 16 | 17 | provider "azurerm" { 18 | features {} 19 | } 20 | 21 | module "resource_group" { 22 | source = "./modules/resource_group" 23 | 24 | resource_group_name = var.resource_group_name 25 | location = var.location 26 | } 27 | 28 | module "network" { 29 | source = "./modules/network" 30 | 31 | resource_group_name = module.resource_group.resource_group_name 32 | location = var.location 33 | vnet_name = var.vnet_name 34 | vnet_address_space = var.vnet_address_space 35 | subnet_name = var.subnet_name 36 | subnet_address_prefixes = var.subnet_address_prefixes 37 | public_ip_name = var.public_ip_name 38 | network_interface_name = var.network_interface_name 39 | nsg_name = var.nsg_name 40 | security_rules = var.security_rules 41 | } 42 | 43 | module "compute" { 44 | source = "./modules/compute" 45 | 46 | vm_name = var.vm_name 47 | resource_group_name = module.resource_group.resource_group_name 48 | location = var.location 49 | vm_size = var.vm_size 50 | admin_username = var.admin_username 51 | network_interface_id = module.network.nic_id 52 | os_disk_storage_type = var.os_disk_storage_type 53 | os_disk_size_gb = var.os_disk_size_gb 54 | image_publisher = var.image_publisher 55 | image_offer = var.image_offer 56 | image_sku = var.image_sku 57 | image_version = var.image_version 58 | cloud_init_file = var.cloud_init_file 59 | ssh_private_key_file = var.ssh_private_key_file 60 | enable_gpu_driver = var.enable_gpu_driver 61 | } 62 | -------------------------------------------------------------------------------- /scripts/azure/terraform/modules/compute/main.tf: -------------------------------------------------------------------------------- 1 | resource "tls_private_key" "vm_key" { 2 | algorithm = "RSA" 3 | rsa_bits = 4096 4 | } 5 | 6 | resource "azurerm_linux_virtual_machine" "vm" { 7 | name = var.vm_name 8 | resource_group_name = var.resource_group_name 9 | location = var.location 10 | size = var.vm_size 11 | admin_username = var.admin_username 12 | network_interface_ids = [ var.network_interface_id ] 13 | 14 | admin_ssh_key { 15 | username = var.admin_username 16 | public_key = tls_private_key.vm_key.public_key_openssh 17 | } 18 | 19 | disable_password_authentication = true 20 | 21 | os_disk { 22 | caching = "ReadWrite" 23 | storage_account_type = var.os_disk_storage_type 24 | disk_size_gb = var.os_disk_size_gb 25 | } 26 | 27 | source_image_reference { 28 | publisher = var.image_publisher 29 | offer = var.image_offer 30 | sku = var.image_sku 31 | version = var.image_version 32 | } 33 | 34 | custom_data = base64encode(file(var.cloud_init_file)) 35 | } 36 | 37 | resource "azurerm_virtual_machine_extension" "gpu_driver" { 38 | count = var.enable_gpu_driver ? 1 : 0 39 | name = "NvidiaGpuDriverLinux" 40 | virtual_machine_id = azurerm_linux_virtual_machine.vm.id 41 | publisher = "Microsoft.HpcCompute" 42 | type = "NvidiaGpuDriverLinux" 43 | type_handler_version = "1.9" 44 | auto_upgrade_minor_version = true 45 | } 46 | 47 | resource "local_file" "ssh_private_key_file" { 48 | content = tls_private_key.vm_key.private_key_pem 49 | filename = var.ssh_private_key_file 50 | file_permission = "0600" 51 | } 52 | -------------------------------------------------------------------------------- /scripts/azure/terraform/modules/compute/outputs.tf: -------------------------------------------------------------------------------- 1 | output "ssh_private_key" { 2 | description = "Private SSH key for accessing the VM" 3 | value = tls_private_key.vm_key.private_key_pem 4 | sensitive = true 5 | } 6 | -------------------------------------------------------------------------------- /scripts/azure/terraform/modules/compute/variables.tf: -------------------------------------------------------------------------------- 1 | variable "vm_name" { 2 | description = "Name of the Virtual Machine" 3 | type = string 4 | } 5 | 6 | variable "resource_group_name" { 7 | description = "Name of the Resource Group" 8 | type = string 9 | } 10 | 11 | variable "location" { 12 | description = "Azure region" 13 | type = string 14 | } 15 | 16 | variable "vm_size" { 17 | description = "VM size" 18 | type = string 19 | } 20 | 21 | variable "admin_username" { 22 | description = "Admin username for the VM" 23 | type = string 24 | } 25 | 26 | variable "network_interface_id" { 27 | description = "ID of the Network Interface to attach to the VM" 28 | type = string 29 | } 30 | 31 | variable "os_disk_storage_type" { 32 | description = "OS Disk storage account type" 33 | type = string 34 | } 35 | 36 | variable "os_disk_size_gb" { 37 | description = "OS Disk size in GB" 38 | type = number 39 | } 40 | 41 | variable "image_publisher" { 42 | description = "Publisher of the VM image" 43 | type = string 44 | } 45 | 46 | variable "image_offer" { 47 | description = "Offer of the VM image" 48 | type = string 49 | } 50 | 51 | variable "image_sku" { 52 | description = "SKU of the VM image" 53 | type = string 54 | } 55 | 56 | variable "image_version" { 57 | description = "Version of the VM image" 58 | type = string 59 | } 60 | 61 | variable "enable_gpu_driver" { 62 | description = "Set to true to enable the NVIDIA GPU driver extension for GPU VMs" 63 | type = bool 64 | default = false 65 | } 66 | 67 | variable "cloud_init_file" { 68 | description = "Path to the cloud-init file" 69 | type = string 70 | } 71 | 72 | variable "ssh_private_key_file" { 73 | description = "File path to save the generated SSH private key" 74 | type = string 75 | } 76 | -------------------------------------------------------------------------------- /scripts/azure/terraform/modules/network/main.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_virtual_network" "vnet" { 2 | name = var.vnet_name 3 | address_space = var.vnet_address_space 4 | location = var.location 5 | resource_group_name = var.resource_group_name 6 | } 7 | 8 | resource "azurerm_subnet" "subnet" { 9 | name = var.subnet_name 10 | resource_group_name = var.resource_group_name 11 | virtual_network_name = azurerm_virtual_network.vnet.name 12 | address_prefixes = var.subnet_address_prefixes 13 | } 14 | 15 | resource "azurerm_public_ip" "pip" { 16 | name = var.public_ip_name 17 | location = var.location 18 | resource_group_name = var.resource_group_name 19 | allocation_method = "Static" 20 | sku = "Standard" 21 | } 22 | 23 | resource "azurerm_network_interface" "nic" { 24 | name = var.network_interface_name 25 | location = var.location 26 | resource_group_name = var.resource_group_name 27 | 28 | ip_configuration { 29 | name = "internal" 30 | subnet_id = azurerm_subnet.subnet.id 31 | private_ip_address_allocation = "Dynamic" 32 | public_ip_address_id = azurerm_public_ip.pip.id 33 | } 34 | } 35 | 36 | resource "azurerm_network_security_group" "nsg" { 37 | name = var.nsg_name 38 | location = var.location 39 | resource_group_name = var.resource_group_name 40 | 41 | dynamic "security_rule" { 42 | for_each = var.security_rules 43 | content { 44 | name = security_rule.value.name 45 | priority = security_rule.value.priority 46 | direction = security_rule.value.direction 47 | access = security_rule.value.access 48 | protocol = security_rule.value.protocol 49 | source_port_range = security_rule.value.source_port_range 50 | destination_port_range = security_rule.value.destination_port_range 51 | source_address_prefix = security_rule.value.source_address_prefix 52 | destination_address_prefix = security_rule.value.destination_address_prefix 53 | } 54 | } 55 | } 56 | 57 | resource "azurerm_network_interface_security_group_association" "nsg_assoc" { 58 | network_interface_id = azurerm_network_interface.nic.id 59 | network_security_group_id = azurerm_network_security_group.nsg.id 60 | } 61 | -------------------------------------------------------------------------------- /scripts/azure/terraform/modules/network/outputs.tf: -------------------------------------------------------------------------------- 1 | output "public_ip" { 2 | value = azurerm_public_ip.pip.ip_address 3 | } 4 | 5 | output "nic_id" { 6 | value = azurerm_network_interface.nic.id 7 | } 8 | -------------------------------------------------------------------------------- /scripts/azure/terraform/modules/network/variables.tf: -------------------------------------------------------------------------------- 1 | variable "resource_group_name" { 2 | description = "Name of the Resource Group" 3 | type = string 4 | } 5 | 6 | variable "location" { 7 | description = "Azure region" 8 | type = string 9 | } 10 | 11 | variable "vnet_name" { 12 | description = "Name of the Virtual Network" 13 | type = string 14 | } 15 | 16 | variable "vnet_address_space" { 17 | description = "Address space for the Virtual Network" 18 | type = list(string) 19 | } 20 | 21 | variable "subnet_name" { 22 | description = "Name of the Subnet" 23 | type = string 24 | } 25 | 26 | variable "subnet_address_prefixes" { 27 | description = "Address prefixes for the Subnet" 28 | type = list(string) 29 | } 30 | 31 | variable "public_ip_name" { 32 | description = "Name of the Public IP" 33 | type = string 34 | } 35 | 36 | variable "network_interface_name" { 37 | description = "Name of the Network Interface" 38 | type = string 39 | } 40 | 41 | variable "nsg_name" { 42 | description = "Name of the Network Security Group" 43 | type = string 44 | } 45 | 46 | variable "security_rules" { 47 | description = "List of NSG security rules" 48 | type = list(object({ 49 | name = string 50 | priority = number 51 | direction = string 52 | access = string 53 | protocol = string 54 | source_port_range = string 55 | destination_port_range = string 56 | source_address_prefix = string 57 | destination_address_prefix = string 58 | })) 59 | } 60 | -------------------------------------------------------------------------------- /scripts/azure/terraform/modules/resource_group/main.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "rg" { 2 | name = var.resource_group_name 3 | location = var.location 4 | } 5 | -------------------------------------------------------------------------------- /scripts/azure/terraform/modules/resource_group/outputs.tf: -------------------------------------------------------------------------------- 1 | output "resource_group_name" { 2 | value = azurerm_resource_group.rg.name 3 | } 4 | -------------------------------------------------------------------------------- /scripts/azure/terraform/modules/resource_group/variables.tf: -------------------------------------------------------------------------------- 1 | variable "resource_group_name" { 2 | description = "Name of the Resource Group" 3 | type = string 4 | } 5 | 6 | variable "location" { 7 | description = "Azure region" 8 | type = string 9 | } 10 | -------------------------------------------------------------------------------- /scripts/azure/terraform/outputs.tf: -------------------------------------------------------------------------------- 1 | output "subscription_id" { 2 | description = "Subscription ID used for the deployment" 3 | value = data.azurerm_client_config.current.subscription_id 4 | } 5 | 6 | output "vm_public_ip" { 7 | description = "Public IP address of the VM" 8 | value = module.network.public_ip 9 | } 10 | 11 | output "ssh_private_key" { 12 | description = "Private SSH key for accessing the VM. Share this securely." 13 | value = module.compute.ssh_private_key 14 | sensitive = true 15 | } 16 | -------------------------------------------------------------------------------- /scripts/check_sizes_of_models.py: -------------------------------------------------------------------------------- 1 | # Use this to scan your huggingface directory and print out the size 2 | # of each model 3 | 4 | import os 5 | 6 | 7 | def get_dir_size(path): 8 | total = 0 9 | if not os.path.exists(path): 10 | return total 11 | with os.scandir(path) as it: 12 | for entry in it: 13 | if entry.is_file(): 14 | total += entry.stat().st_size 15 | elif entry.is_dir(): 16 | total += get_dir_size(entry.path) 17 | return total 18 | 19 | 20 | cache_dir = os.path.expanduser("~/.cache/huggingface/hub") 21 | for entry in os.scandir(cache_dir): 22 | if entry.is_dir(): 23 | blobs_dir = os.path.join(entry.path, "blobs") 24 | if not os.path.isdir(blobs_dir): 25 | continue 26 | 27 | dir_size = get_dir_size(blobs_dir) / 1024 / 1024 28 | # print(f"{entry.name}: {dir_size:.1f} MB") 29 | 30 | # obtain the model name by getting the part of entry after the last "--": 31 | model_name = entry.name.split("--")[-1] 32 | print(f"{model_name}\n{dir_size:.1f}\n") 33 | -------------------------------------------------------------------------------- /scripts/conda_activate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ENV_NAME="transformerlab" 3 | TLAB_DIR="$HOME/.transformerlab" 4 | TLAB_CODE_DIR="${TLAB_DIR}/src" 5 | 6 | MINIFORGE_ROOT=${TLAB_DIR}/miniforge3 7 | CONDA_BIN=${MINIFORGE_ROOT}/bin/conda 8 | ENV_DIR=${TLAB_DIR}/envs/${ENV_NAME} 9 | 10 | echo "Your shell is $SHELL" 11 | echo "Conda's binary is at ${CONDA_BIN}" 12 | echo "Your current directory is $(pwd)" 13 | 14 | err_report() { 15 | echo "Error in run.sh on line $1" 16 | } 17 | 18 | trap 'err_report $LINENO' ERR 19 | 20 | if ! command -v ${CONDA_BIN} &> /dev/null; then 21 | echo "❌ Conda is not installed at ${MINIFORGE_ROOT}. Please install Conda there (and only there) and try again." 22 | else 23 | echo "✅ Conda is installed." 24 | fi 25 | 26 | echo "👏 Enabling conda in shell" 27 | 28 | eval "$(${CONDA_BIN} shell.bash hook)" 29 | 30 | echo "👏 Activating transformerlab conda environment" 31 | conda activate "${ENV_DIR}" -------------------------------------------------------------------------------- /scripts/xml-rpc-client-example/example.py: -------------------------------------------------------------------------------- 1 | import xmlrpc.client 2 | import json # For pretty-printing the results 3 | 4 | # Connect to the XML-RPC server 5 | server = xmlrpc.client.ServerProxy("http://localhost:8338/job_sdk") 6 | 7 | # Get a simple string response 8 | hello_result = server.hello("XML-RPC User") 9 | print(f"String result: {hello_result}") 10 | 11 | # Get a dictionary/object response 12 | user = server.get_user(1) 13 | print("\nDictionary result:") 14 | print(json.dumps(user, indent=2)) # Pretty print the dictionary 15 | 16 | # Get a list of objects 17 | users = server.list_users() 18 | print("\nList of objects result:") 19 | print(json.dumps(users, indent=2)) 20 | 21 | # Get a complex nested object 22 | project = server.get_project_data(123) 23 | print("\nComplex nested object result:") 24 | print(json.dumps(project, indent=2)) 25 | 26 | # Check what happens with non-existent data 27 | missing_user = server.get_user(999) 28 | print("\nNon-existent data result:") 29 | print(json.dumps(missing_user, indent=2)) 30 | 31 | # We can also work with the returned objects directly 32 | if user.get("active", False): 33 | print(f"\nUser {user['name']} is active!") 34 | else: 35 | print(f"\nUser {user['name']} is not active!") 36 | 37 | # Calculate average progress of tasks in a project 38 | if "tasks" in project: 39 | complete_tasks = sum(1 for task in project["tasks"] if task["complete"]) 40 | total_tasks = len(project["tasks"]) 41 | print(f"\nProject task completion: {complete_tasks}/{total_tasks} ({complete_tasks / total_tasks * 100:.0f}%)") 42 | -------------------------------------------------------------------------------- /scripts/xml-rpc-client-example/tlab_sdk_client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/scripts/xml-rpc-client-example/tlab_sdk_client/__init__.py -------------------------------------------------------------------------------- /scripts/xml-rpc-client-example/tlab_sdk_client/callbacks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/scripts/xml-rpc-client-example/tlab_sdk_client/callbacks/__init__.py -------------------------------------------------------------------------------- /scripts/xml-rpc-client-example/tlab_sdk_client/callbacks/hf_callback.py: -------------------------------------------------------------------------------- 1 | from transformers import TrainerCallback 2 | 3 | 4 | class TLabProgressCallback(TrainerCallback): 5 | def __init__(self, tlab_client): 6 | self.tlab_client = tlab_client 7 | 8 | def on_step_end(self, args, state, control, **kwargs): 9 | if state.is_local_process_zero: 10 | if state.max_steps > 0: 11 | # Calculate progress percentage (30-90%) 12 | progress = 30 + ((state.global_step / state.max_steps) * 60) 13 | metrics = { 14 | "step": state.global_step, 15 | "train/loss": state.log_history[-1]["loss"] if state.log_history else None, 16 | } 17 | # Report progress to TransformerLab 18 | if not self.tlab_client.report_progress(progress, metrics): 19 | # Job was stopped remotely 20 | control.should_training_stop = True 21 | 22 | def on_log(self, args, state, control, logs=None, **kwargs): 23 | if logs and "loss" in logs: 24 | metrics = {"step": state.global_step, "train/loss": logs["loss"]} 25 | # Add other metrics if available 26 | for key, value in logs.items(): 27 | if isinstance(value, (int, float)): 28 | metrics[key] = value 29 | self.tlab_client.report_progress(30 + ((state.global_step / state.max_steps) * 60), metrics) 30 | -------------------------------------------------------------------------------- /scripts/xml-rpc-client-example/train_example.py: -------------------------------------------------------------------------------- 1 | import xmlrpc.client 2 | import json 3 | import time 4 | 5 | # Connect to the XML-RPC server 6 | server = xmlrpc.client.ServerProxy("http://localhost:8338") 7 | 8 | # Configuration for a training job 9 | training_config = { 10 | "experiment_id": "alpha", 11 | "model_name": "mlx-community/Llama-3.2-1B-Instruct-4bit", 12 | "dataset": "Trelis/touch-rugby-rules", 13 | "template_name": "default", 14 | "output_dir": "./output", 15 | "log_to_wandb": False, 16 | "_config": { 17 | "dataset_name": "Trelis/touch-rugby-rules", 18 | "lr": 3e-5, 19 | "num_train_epochs": 3, 20 | "batch_size": 8, 21 | "gradient_accumulation_steps": 1, 22 | }, 23 | } 24 | 25 | # Start a training job 26 | # job_id = "998" 27 | result = server.start_training(json.dumps(training_config)) 28 | job_id = result["job_id"] 29 | print(f"\nTraining job ID: {job_id}") 30 | print("Started training job:") 31 | print(json.dumps(result, indent=2)) 32 | 33 | # Poll for status updates 34 | for i in range(10): 35 | time.sleep(5) 36 | status = server.get_training_status(job_id, (i + 1) * 10) 37 | print(f"\nTraining status at {time.strftime('%H:%M:%S')}:") 38 | print(json.dumps(status, indent=2)) 39 | 40 | # Break if training is complete 41 | if status["status"] in ["COMPLETE", "FAILED", "STOPPED"]: 42 | break 43 | 44 | # Optionally stop the training 45 | stop_result = server.stop_training(job_id) 46 | print("\nStopped training job:") 47 | print(json.dumps(stop_result, indent=2)) 48 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | ## Testing 2 | 3 | So far we have the following PyTests: 4 | 5 | * `db/*` These are tests for the database layer 6 | * `plugins/*` In here is a test to ensure all plugin `index.json` follow the correct plugin schema 7 | * `api/*` here are all the tests for the API. Make sure you activate the conda environment before running these tests. 8 | 9 | ### Running the Tests 10 | 11 | Run these from the root of the project. 12 | 13 | To run the faster non-API tests: 14 | ``` 15 | uv pip install --system pytest pytest-asyncio jsonschema shellcheck-py 16 | pytest test/db/ test/plugins/ 17 | ``` 18 | 19 | To run the slower API tests: 20 | ``` 21 | uv pip install --system pytest pytest-asyncio jsonschema requests shellcheck-py 22 | pytest test/api/ 23 | ``` -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/test/__init__.py -------------------------------------------------------------------------------- /test/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/test/api/__init__.py -------------------------------------------------------------------------------- /test/api/conftest.py: -------------------------------------------------------------------------------- 1 | # No longer needed: live_server fixture for external server. All tests now use TestClient in-process. 2 | # This file can be deleted or left empty. 3 | -------------------------------------------------------------------------------- /test/api/test_api.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api import app 3 | 4 | 5 | def test_root(): 6 | with TestClient(app) as client: 7 | response = client.get("/") 8 | assert response.status_code == 200 9 | -------------------------------------------------------------------------------- /test/api/test_config.py: -------------------------------------------------------------------------------- 1 | # Test FastAPI endpoints using TestClient (in-process, for coverage) 2 | from fastapi.testclient import TestClient 3 | 4 | from api import app 5 | 6 | client = TestClient(app) 7 | 8 | 9 | def test_set_config(): 10 | with TestClient(app) as client: 11 | response = client.get("/config/set", params={"k": "api_test_key", "v": "test_value"}) 12 | assert response.status_code == 200 13 | assert response.json() == {"key": "api_test_key", "value": "test_value"} 14 | 15 | 16 | def test_get_config(): 17 | # Ensure the value is set first 18 | with TestClient(app) as client: 19 | client.get("/config/set", params={"k": "api_test_key2", "v": "test_value2"}) 20 | response = client.get("/config/get/api_test_key2") 21 | assert response.status_code == 200 22 | assert response.json() == "test_value2" 23 | -------------------------------------------------------------------------------- /test/api/test_evals.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api import app 3 | 4 | 5 | def test_evals_list(): 6 | with TestClient(app) as client: 7 | resp = client.get("/evals/list") 8 | assert resp.status_code == 200 9 | data = resp.json() 10 | assert isinstance(data, list) 11 | if data: 12 | eval_item = data[0] 13 | assert "name" in eval_item or "info" in eval_item 14 | 15 | 16 | def test_evals_compare(): 17 | with TestClient(app) as client: 18 | resp = client.get("/evals/compare_evals?job_list=1,2,3") 19 | assert resp.status_code in (200, 400, 404) 20 | -------------------------------------------------------------------------------- /test/api/test_plugins.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api import app 3 | 4 | 5 | def test_plugins_gallery(): 6 | with TestClient(app) as client: 7 | resp = client.get("/plugins/gallery") 8 | assert resp.status_code == 200 9 | data = resp.json() 10 | assert isinstance(data, list) 11 | if data: 12 | plugin = data[0] 13 | assert "name" in plugin or "description" in plugin 14 | 15 | 16 | def test_plugins_list(): 17 | with TestClient(app) as client: 18 | resp = client.get("/plugins/list") 19 | assert resp.status_code == 200 20 | data = resp.json() 21 | assert isinstance(data, list) 22 | if data: 23 | plugin = data[0] 24 | assert "name" in plugin or "description" in plugin 25 | 26 | 27 | def test_plugins_install(): 28 | with TestClient(app) as client: 29 | resp = client.get("/plugins/gallery/fastchat_server/install") 30 | assert resp.status_code in (200, 404) 31 | if resp.status_code == 200: 32 | data = resp.json() 33 | assert "message" in data or "status" in data 34 | 35 | 36 | def test_run_installer_script(): 37 | with TestClient(app) as client: 38 | resp = client.get("/plugins/fastchat_server/run_installer_script") 39 | # Installer may not exist, so allow 200 or 404 40 | assert resp.status_code in (200, 404) 41 | if resp.status_code == 200: 42 | data = resp.json() 43 | assert "message" in data or "status" in data 44 | 45 | 46 | def test_list_missing_plugins_for_current_platform(): 47 | with TestClient(app) as client: 48 | resp = client.get("/plugins/list_missing_plugins_for_current_platform") 49 | assert resp.status_code == 200 50 | data = resp.json() 51 | assert isinstance(data, list) 52 | 53 | 54 | def test_install_missing_plugins_for_current_platform(): 55 | with TestClient(app) as client: 56 | resp = client.get("/plugins/install_missing_plugins_for_current_platform") 57 | assert resp.status_code == 200 58 | data = resp.json() 59 | assert isinstance(data, list) 60 | 61 | 62 | def test_autoupdate_all_plugins(): 63 | with TestClient(app) as client: 64 | resp = client.get("/plugins/autoupdate_all_plugins") 65 | assert resp.status_code in (200, 404) 66 | if resp.status_code == 200: 67 | data = resp.json() 68 | assert "message" in data or "status" in data 69 | -------------------------------------------------------------------------------- /test/api/test_prompts.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api import app 3 | 4 | 5 | def test_prompts_list(): 6 | with TestClient(app) as client: 7 | resp = client.get("/prompts/list") 8 | assert resp.status_code == 200 9 | assert isinstance(resp.json(), list) or isinstance(resp.json(), dict) 10 | 11 | 12 | def test_prompts_dummy(): 13 | with TestClient(app) as client: 14 | resp = client.get("/prompts/list?prompt_id=dummy") 15 | assert resp.status_code in (200, 400, 404) 16 | -------------------------------------------------------------------------------- /test/api/test_server_info.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api import app 3 | 4 | 5 | def test_server_info(): 6 | with TestClient(app) as client: 7 | response = client.get("/server/info") 8 | assert response.status_code == 200 9 | data = response.json() 10 | assert isinstance(data, dict) 11 | assert "cpu" in data 12 | assert "memory" in data and isinstance(data["memory"], dict) 13 | assert "disk" in data and isinstance(data["disk"], dict) 14 | assert "gpu" in data 15 | mem = data["memory"] 16 | for key in ("total", "available", "percent", "used", "free"): 17 | assert key in mem 18 | disk = data["disk"] 19 | for key in ("total", "used", "free", "percent"): 20 | assert key in disk 21 | 22 | 23 | def test_server_info_keys(): 24 | with TestClient(app) as client: 25 | response = client.get("/server/info") 26 | assert response.status_code == 200 27 | data = response.json() 28 | # Check for some extra keys 29 | for key in ["pytorch_version", "flash_attn_version", "device", "device_type", "os", "python_version"]: 30 | assert key in data 31 | # If running on Mac, check for mac_metrics (may be None) 32 | import sys 33 | 34 | if sys.platform == "darwin": 35 | # mac_metrics may or may not be present, but if present, should be a dict 36 | if "mac_metrics" in data: 37 | assert isinstance(data["mac_metrics"], dict) 38 | 39 | 40 | def test_server_python_libraries(): 41 | with TestClient(app) as client: 42 | response = client.get("/server/python_libraries") 43 | assert response.status_code == 200 44 | data = response.json() 45 | assert isinstance(data, list) 46 | assert len(data) > 0 47 | for package in data: 48 | assert isinstance(package, dict) 49 | assert "name" in package and isinstance(package["name"], str) and package["name"] 50 | assert "version" in package and isinstance(package["version"], str) and package["version"] 51 | 52 | 53 | def test_server_pytorch_collect_env(): 54 | with TestClient(app) as client: 55 | response = client.get("/server/pytorch_collect_env") 56 | assert response.status_code == 200 57 | data = response.text 58 | assert "PyTorch" in data 59 | 60 | 61 | def test_is_wsl_false(monkeypatch): 62 | # Simulate subprocess.CalledProcessError 63 | import subprocess 64 | 65 | def fake_check_output(*args, **kwargs): 66 | raise subprocess.CalledProcessError(1, "uname") 67 | 68 | monkeypatch.setattr(subprocess, "check_output", fake_check_output) 69 | from transformerlab.routers import serverinfo 70 | 71 | assert serverinfo.is_wsl() is False 72 | -------------------------------------------------------------------------------- /test/api/test_tasks.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api import app 3 | 4 | 5 | def test_tasks_list(): 6 | with TestClient(app) as client: 7 | resp = client.get("/tasks/list") 8 | assert resp.status_code == 200 9 | assert isinstance(resp.json(), list) or isinstance(resp.json(), dict) 10 | 11 | 12 | def test_tasks_get_by_id(): 13 | with TestClient(app) as client: 14 | resp = client.get("/tasks/1/get") 15 | assert resp.status_code in (200, 404) 16 | 17 | 18 | def test_tasks_list_by_type(): 19 | with TestClient(app) as client: 20 | resp = client.get("/tasks/list_by_type?type=TRAIN") 21 | assert resp.status_code in (200, 404) 22 | 23 | 24 | def test_add_task(): 25 | with TestClient(app) as client: 26 | new_task = { 27 | "name": "Test Task", 28 | "type": "TRAIN", 29 | "inputs": "{}", 30 | "config": "{}", 31 | "plugin": "test_plugin", 32 | "outputs": "{}", 33 | "experiment_id": 1, 34 | } 35 | resp = client.put("/tasks/new_task", json=new_task) 36 | assert resp.status_code == 200 37 | assert "message" in resp.json() or "status" in resp.json() 38 | 39 | 40 | def test_update_task(): 41 | with TestClient(app) as client: 42 | update_data = {"name": "Updated Task", "inputs": "{}", "config": "{}", "outputs": "{}"} 43 | resp = client.put("/tasks/1/update", json=update_data) 44 | assert resp.status_code == 200 45 | assert resp.json()["message"] == "OK" 46 | 47 | 48 | def test_list_by_type_in_experiment(): 49 | with TestClient(app) as client: 50 | resp = client.get("/tasks/list_by_type_in_experiment?type=TRAIN&experiment_id=1") 51 | assert resp.status_code in (200, 404) 52 | 53 | 54 | def test_delete_task(): 55 | with TestClient(app) as client: 56 | resp = client.get("/tasks/1/delete") 57 | assert resp.status_code == 200 58 | assert resp.json()["message"] == "OK" 59 | 60 | 61 | def test_delete_all_tasks(): 62 | with TestClient(app) as client: 63 | resp = client.get("/tasks/delete_all") 64 | assert resp.status_code == 200 65 | assert resp.json()["message"] == "OK" 66 | -------------------------------------------------------------------------------- /test/api/test_tools.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api import app 3 | 4 | 5 | def test_tools_list(): 6 | with TestClient(app) as client: 7 | resp = client.get("/tools/list") 8 | assert resp.status_code == 200 9 | assert isinstance(resp.json(), list) or isinstance(resp.json(), dict) 10 | 11 | 12 | def test_tools_call(): 13 | with TestClient(app) as client: 14 | resp = client.get("/tools/call/add?params={}") 15 | assert resp.status_code in (200, 400, 404) 16 | 17 | 18 | def test_tools_prompt(): 19 | with TestClient(app) as client: 20 | resp = client.get("/tools/prompt") 21 | assert resp.status_code == 200 22 | assert "" in resp.text 23 | 24 | 25 | def test_tools_call_invalid_tool(): 26 | with TestClient(app) as client: 27 | resp = client.get("/tools/call/invalid_tool?params={}") 28 | assert resp.status_code == 200 29 | assert resp.json()["status"] == "error" 30 | 31 | 32 | def test_tools_call_invalid_params(): 33 | with TestClient(app) as client: 34 | resp = client.get("/tools/call/add?params=not_a_json") 35 | assert resp.status_code == 200 36 | assert resp.json()["status"] == "error" 37 | 38 | 39 | def test_tools_install_mcp_server_invalid_file(): 40 | with TestClient(app) as client: 41 | resp = client.get("/tools/install_mcp_server?server_name=/not/a/real/path.py") 42 | assert resp.status_code == 403 43 | assert resp.json()["status"] == "error" 44 | -------------------------------------------------------------------------------- /test/api/test_train.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api import app 3 | 4 | 5 | def test_train_templates(): 6 | with TestClient(app) as client: 7 | resp = client.get("/train/templates") 8 | assert resp.status_code == 200 9 | assert isinstance(resp.json(), list) or isinstance(resp.json(), dict) 10 | 11 | 12 | def test_train_recipe_gallery(): 13 | with TestClient(app) as client: 14 | resp = client.get("/train/template/gallery") 15 | assert resp.status_code == 200 16 | assert isinstance(resp.json(), list) or isinstance(resp.json(), dict) 17 | 18 | 19 | def test_train_export_recipe(): 20 | with TestClient(app) as client: 21 | resp = client.get("/train/template/1/export") 22 | assert resp.status_code in (200, 404) 23 | 24 | 25 | def test_train_create_template(): 26 | with TestClient(app) as client: 27 | data = {"name": "test_template", "description": "desc", "type": "test", "config": "{}"} 28 | resp = client.post("/train/template/create", data=data) 29 | assert resp.status_code in (200, 422, 400) 30 | -------------------------------------------------------------------------------- /test/api/test_user.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api import app 3 | 4 | 5 | def test_register_user(): 6 | with TestClient(app) as client: 7 | new_user = { 8 | "email": "pytest@test.com", 9 | "password": "password", 10 | } 11 | resp = client.post("/auth/register", json=new_user) 12 | 13 | # Success = HTTP 201 with a field called "id" 14 | # Already exists = HTTP 400 with a field called "detail" 15 | # (vs. a bad request which is HTTP 400 and a FastAPI error) 16 | assert resp.status_code in (201, 400) 17 | resp_json = resp.json() 18 | print(resp_json) 19 | assert "id" in resp_json or resp_json.get("detail", "") == "REGISTER_USER_ALREADY_EXISTS" 20 | -------------------------------------------------------------------------------- /test/api/test_workflows.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api import app 3 | 4 | 5 | def test_workflows_list(): 6 | with TestClient(app) as client: 7 | resp = client.get("/workflows/list") 8 | assert resp.status_code == 200 9 | assert isinstance(resp.json(), list) or isinstance(resp.json(), dict) 10 | 11 | 12 | def test_workflows_delete(): 13 | with TestClient(app) as client: 14 | resp = client.get("/workflows/delete/dummy_workflow") 15 | assert resp.status_code in (200, 404) 16 | 17 | 18 | def test_workflows_create(): 19 | with TestClient(app) as client: 20 | resp = client.get("/workflows/create?name=test_workflow") 21 | assert resp.status_code in (200, 400, 404) 22 | -------------------------------------------------------------------------------- /test/db/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/test/db/__init__.py -------------------------------------------------------------------------------- /test/installer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/test/installer/__init__.py -------------------------------------------------------------------------------- /test/installer/test_install_dot_sh.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from pathlib import Path 3 | 4 | 5 | def test_install_sh_shellcheck(): 6 | install_sh = Path(__file__).parent.parent.parent / "install.sh" 7 | assert install_sh.exists(), "install.sh not found in repository root" 8 | 9 | result = subprocess.run( 10 | ["shellcheck", "--severity=error", str(install_sh)], 11 | capture_output=True, 12 | text=True, 13 | ) 14 | 15 | assert result.returncode == 0, f"shellcheck found errors:\n{result.stdout}\n{result.stderr}" 16 | -------------------------------------------------------------------------------- /test/plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/test/plugins/__init__.py -------------------------------------------------------------------------------- /test/plugins/plugin.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "type": "object", 4 | "properties": { 5 | "name": { "type": "string" }, 6 | "uniqueId": { "type": "string" }, 7 | "description": { "type": "string" }, 8 | "plugin-format": { "type": "string" }, 9 | "type": { 10 | "type": "string", 11 | "enum": [ 12 | "loader", 13 | "trainer", 14 | "evaluator", 15 | "generator", 16 | "eval", 17 | "exporter", 18 | "rag" 19 | ] 20 | }, 21 | "version": { "type": "string" }, 22 | "model_architectures": { 23 | "type": "array", 24 | "items": { "type": "string" } 25 | }, 26 | "supported_hardware_architectures": { 27 | "type": "array", 28 | "items": { 29 | "type": "string", 30 | "enum": ["cuda", "mlx", "cpu", "amd"] 31 | } 32 | }, 33 | "files": { 34 | "type": "array", 35 | "items": { "type": "string" } 36 | }, 37 | "setup-script": { "type": "string" }, 38 | "parameters": { "type": "object" }, 39 | "parameters_ui": { "type": "object" } 40 | }, 41 | "required": [ 42 | "name", 43 | "uniqueId", 44 | "description", 45 | "plugin-format", 46 | "type", 47 | "version", 48 | "files", 49 | "setup-script", 50 | "supported_hardware_architectures" 51 | ], 52 | "additionalProperties": true 53 | } 54 | -------------------------------------------------------------------------------- /test/plugins/test_plugins_schema.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import pytest 4 | from jsonschema import validate, ValidationError 5 | 6 | # Get the directory of the current script 7 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 8 | 9 | # Load the schema using a relative path 10 | SCHEMA_PATH = os.path.join(BASE_DIR, "plugin.schema.json") 11 | with open(SCHEMA_PATH, "r") as schema_file: 12 | SCHEMA = json.load(schema_file) 13 | 14 | # Define the base directory for plugins using a relative path 15 | PLUGINS_DIR = os.path.join(BASE_DIR, "../../transformerlab/plugins") 16 | 17 | 18 | def find_index_json_files(base_dir): 19 | """Find all index.json files in the first level of each folder under the given directory.""" 20 | for entry in os.scandir(base_dir): 21 | if entry.is_dir(): 22 | index_file = os.path.join(entry.path, "index.json") 23 | if os.path.isfile(index_file): 24 | yield index_file 25 | 26 | 27 | @pytest.mark.parametrize("index_json_path", find_index_json_files(PLUGINS_DIR)) 28 | def test_validate_index_json(index_json_path): 29 | """Validate each index.json file against the schema.""" 30 | with open(index_json_path, "r") as json_file: 31 | try: 32 | data = json.load(json_file) 33 | validate(instance=data, schema=SCHEMA) 34 | except ValidationError as e: 35 | pytest.fail(f"Validation failed for {index_json_path}: {e.message}") 36 | except json.JSONDecodeError as e: 37 | pytest.fail(f"Invalid JSON in {index_json_path}: {e.msg}") 38 | -------------------------------------------------------------------------------- /test/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/test/server/__init__.py -------------------------------------------------------------------------------- /test/server/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture(scope="session") 5 | def live_server(): 6 | # Get a free port 7 | # import socket 8 | 9 | # s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 10 | # s.bind(("127.0.0.1", 0)) 11 | # port = s.getsockname()[1] 12 | # s.close() 13 | 14 | # # Start the server process 15 | # print("about to run: ./run.sh -p", port) 16 | host = "127.0.0.1" 17 | port = 8338 # For testing, we can use a fixed port 18 | # server_process = subprocess.Popen(["./run.sh", "-h", host, "-p", str(port)]) 19 | 20 | # # Give it time to start 21 | # time.sleep(20) 22 | 23 | base_url = f"http://{host}:{port}" 24 | 25 | # Verify the server is running 26 | import requests 27 | 28 | try: 29 | response = requests.get(f"{base_url}/") 30 | assert response.status_code == 200 31 | except Exception as e: 32 | # server_process.terminate() 33 | raise Exception(f"Failed to start server: {e}") 34 | 35 | yield base_url 36 | 37 | # Teardown - stop the server 38 | # server_process.terminate() 39 | # server_process.wait() 40 | -------------------------------------------------------------------------------- /test/server/test_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import requests 3 | 4 | 5 | @pytest.mark.live_server 6 | def test_set(live_server): 7 | response = requests.get(f"{live_server}/config/set", params={"k": "message", "v": "Hello, World!"}) 8 | assert response.status_code == 200 9 | assert response.json() == {"key": "message", "value": "Hello, World!"} 10 | 11 | 12 | @pytest.mark.live_server 13 | def test_get(live_server): 14 | response = requests.get(f"{live_server}/config/get/message") 15 | assert response.status_code == 200 16 | assert response.json() == "Hello, World!" 17 | -------------------------------------------------------------------------------- /test/server/test_server_info.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import requests 3 | 4 | 5 | @pytest.mark.live_server 6 | def test_server_info(live_server): 7 | response = requests.get(f"{live_server}/server/info") 8 | assert response.status_code == 200 9 | data = response.json() 10 | assert isinstance(data, dict) 11 | assert "cpu" in data 12 | assert "memory" in data and isinstance(data["memory"], dict) 13 | assert "disk" in data and isinstance(data["disk"], dict) 14 | assert "gpu" in data 15 | # Check memory fields 16 | mem = data["memory"] 17 | for key in ("total", "available", "percent", "used", "free"): 18 | assert key in mem 19 | # Check disk fields 20 | disk = data["disk"] 21 | for key in ("total", "used", "free", "percent"): 22 | assert key in disk 23 | 24 | 25 | @pytest.mark.live_server 26 | def test_server_python_libraries(live_server): 27 | response = requests.get(f"{live_server}/server/python_libraries") 28 | assert response.status_code == 200 29 | data = response.json() 30 | # assert it is an array of {"name": "package_name", "version": "version_number"} type things 31 | assert isinstance(data, list) 32 | assert len(data) > 0 33 | for package in data: 34 | assert isinstance(package, dict) 35 | assert "name" in package and isinstance(package["name"], str) and package["name"] 36 | assert "version" in package and isinstance(package["version"], str) and package["version"] 37 | 38 | 39 | @pytest.mark.live_server 40 | def test_server_pytorch_collect_env(live_server): 41 | response = requests.get(f"{live_server}/server/pytorch_collect_env") 42 | assert response.status_code == 200 43 | data = response.text 44 | assert "PyTorch" in data 45 | -------------------------------------------------------------------------------- /test/tmp/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore every file in this folder 2 | * 3 | # Except this one 4 | !.gitignore -------------------------------------------------------------------------------- /transformerlab/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/__init__.py -------------------------------------------------------------------------------- /transformerlab/galleries/plugin-gallery.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /transformerlab/galleries/prompt-gallery.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "parse-unstructured-data", 4 | "text": "You are a data scientist tasked with parsing unstructured data. \nGiven the following text, output the structured data.\n\n{text}\n\nStructured Data:\n", 5 | "title": "Parse unstructured data", 6 | "tags": ["conversion"], 7 | "version": "0.1.0" 8 | }, 9 | { 10 | "id": "convert-to-standard-english", 11 | "text": "You will be provided with a statement, and your task is to convert it to standard English.\n\nStatement:\n\n{text}\n\nStandard English:\n", 12 | "title": "Convert any text to standard English", 13 | "parameters": { 14 | "temperature": 0.7, 15 | "max_tokens": 64, 16 | "top_p": 1 17 | }, 18 | "tags": ["translation"], 19 | "version": "0.1.0" 20 | }, 21 | { 22 | "id": "write-a-summary", 23 | "text": "You are a journalist tasked with writing a summary of the following text.\n\n{text}\n\nSummary:\n", 24 | "title": "Summarize text", 25 | "tags": ["summarization"], 26 | "version": "0.1.0" 27 | }, 28 | { 29 | "id": "summarize-for-second-grader", 30 | "text": "Summarize content you are provided with for a second-grade student.\n\nContent:\n\n{text}\n\nSummary:\n", 31 | "title": "Summarize for Second-Grade Student", 32 | "tags": ["summarization"], 33 | "version": "0.1.0" 34 | }, 35 | { 36 | "id": "summarize-transcript", 37 | "text": "Summarize the following chat transcript by shortening and summarizing the content \nwithout losing important information:\n\n{text}\n\nSummary:\n", 38 | "title": "Summarize chat history", 39 | "tags": ["summarization"], 40 | "version": "0.1.0" 41 | }, 42 | { 43 | "id": "convert-csv-to-markdown-table", 44 | "text": "You are an expert in data formatting. For the following csv data, output it as a markdown \ntable.\nOutput the table only.\n```{text}```\n", 45 | "title": "Convert CSV to Markdown Table", 46 | "tags": ["conversion"], 47 | "version": "0.1.0" 48 | } 49 | ] 50 | -------------------------------------------------------------------------------- /transformerlab/launch_header_text.txt: -------------------------------------------------------------------------------- 1 | 2 | _____ __ 3 | |_ _| / _| 4 | | |_ __ __ _ _ __ ___| |_ ___ _ __ _ __ ___ ___ _ __ 5 | | | '__/ _` | '_ \/ __| _/ _ \| '__| '_ ` _ \ / _ \ '__| 6 | | | | | (_| | | | \__ \ || (_) | | | | | | | | __/ | 7 | \_/_| \__,_|_| |_|___/_| \___/|_| |_| |_| |_|\___|_| 8 | _ _ 9 | | | | | 10 | | | __ _| |__ 11 | | | / _` | '_ \ 12 | | |___| (_| | |_) | 13 | \_____/\__,_|_.__/ 14 | -------------------------------------------------------------------------------- /transformerlab/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/models/__init__.py -------------------------------------------------------------------------------- /transformerlab/models/model_helper.py: -------------------------------------------------------------------------------- 1 | """ 2 | model_helper.py 3 | 4 | Common functions for working with models from various sources. 5 | 6 | Most parts of the API will just use this helper and probably 7 | don't have to interact directly with the source and model classes. 8 | """ 9 | 10 | from transformerlab.models import ollamamodel 11 | from transformerlab.models import huggingfacemodel 12 | from transformerlab.models import localmodel 13 | 14 | import traceback 15 | 16 | 17 | async def list_installed_models(embedding=False): 18 | """ 19 | This function checks both the DB and the workspace models directory 20 | and returns a list of models in the format that models are stored in the DB. 21 | """ 22 | return await localmodel.LocalModelStore().list_models(embedding) 23 | 24 | 25 | async def is_model_installed(model_id: str): 26 | """ 27 | Return true is a model with the unique ID model_id 28 | is downloaded to the DB or Transfomrmer Lab workspace. 29 | """ 30 | return await localmodel.LocalModelStore().has_model(model_id) 31 | 32 | 33 | async def list_model_provenance(model_id: str): 34 | """ 35 | Return a list of the provenance of a model with the unique ID model_id. 36 | """ 37 | return await localmodel.LocalModelStore().list_model_provenance(model_id) 38 | 39 | 40 | ### 41 | # EXTERNAL MODEL SOURCE WRAPPER FUNCTIONS 42 | # 43 | # These functions get used primarily when importing models. 44 | ### 45 | 46 | 47 | def list_model_sources(): 48 | """ 49 | Supported strings that can be passsed as model_source 50 | to the functons that follow. 51 | """ 52 | return ["huggingface", "ollama"] 53 | 54 | 55 | def get_model_by_source_id(model_source: str, model_source_id: str): 56 | """ 57 | Get a model from a model_source. 58 | model_source needs to be one of the strings returned by list_model_sources. 59 | model_source_id is the ID for that model internal to the model_source. 60 | """ 61 | 62 | try: 63 | match model_source: 64 | case "ollama": 65 | return ollamamodel.OllamaModel(model_source_id) 66 | case "huggingface": 67 | return huggingfacemodel.HuggingFaceModel(model_source_id) 68 | except Exception: 69 | print(f"Caught exception getting model {model_source_id} from {model_source}:") 70 | traceback.print_exc() 71 | return None 72 | 73 | 74 | async def list_models_from_source(model_source: str): 75 | """ 76 | Get a list of models available at model_source. 77 | model_source needs to be one of the strings returned by list_model_sources. 78 | """ 79 | try: 80 | match model_source: 81 | case "ollama": 82 | return await ollamamodel.list_models() 83 | case "huggingface": 84 | return await huggingfacemodel.list_models() 85 | except Exception: 86 | print(f"Caught exception listing models from {model_source}:") 87 | traceback.print_exc() 88 | return [] 89 | -------------------------------------------------------------------------------- /transformerlab/models/modelstore.py: -------------------------------------------------------------------------------- 1 | """ 2 | ModelStore - Root class that other model stores inherit from. 3 | This is not useful on its own, it just defines the base object. 4 | Sort of like an abstract class or interface. 5 | """ 6 | 7 | 8 | class ModelStore: 9 | def __str__(self): 10 | # For debug output 11 | return str(self.__class__) + ": " + str(self.__dict__) 12 | 13 | async def list_models(self): 14 | return [] 15 | 16 | async def has_model(self, model_id): 17 | """ 18 | Probably don't need to override this. 19 | """ 20 | model_list = await self.list_models() 21 | for model in model_list: 22 | if model["model_id"] == model_id: 23 | return True 24 | # Sort of hack: If create a GGUF file in Transformer Lab it 25 | # will have "TransformerLab/" at the front but some checks 26 | # (for example coming from Ollama) won't have that and we don't 27 | # want to create unnecessary duplicates. So.... 28 | if model["model_id"] == f"TransformerLab/{model_id}": 29 | return True 30 | return False 31 | -------------------------------------------------------------------------------- /transformerlab/plugin_sdk/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/plugin_sdk/__init__.py -------------------------------------------------------------------------------- /transformerlab/plugin_sdk/plugin_harness.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is the main file that gets called by popen when a plugin is run. 3 | It must get passed: 4 | 5 | --plugin_dir full path to the directory containing the plugin 6 | 7 | All other parameters can be passed as if you are calling the plugin directly. 8 | 9 | """ 10 | 11 | import sys 12 | import argparse 13 | import traceback 14 | 15 | 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument("--plugin_dir", type=str, required=True) 18 | args, unknown = parser.parse_known_args() 19 | 20 | # Add the plugin directory to the path 21 | # Note that this will allow the plugin to import files in this file's directory 22 | # So the plugin is able to import the SDK 23 | sys.path.append(args.plugin_dir) 24 | 25 | try: 26 | import main 27 | except ImportError as e: 28 | print(f"Error executing plugin: {e}") 29 | traceback.print_exc() 30 | 31 | # if e is a ModuleNotFoundError, the plugin is missing a required package 32 | if isinstance(e, ModuleNotFoundError): 33 | print("ModuleNotFoundError means a Python package is missing. This is usually fixed by reinstalling the plugin") 34 | 35 | sys.exit(1) 36 | 37 | # Also execute the function main.main(), if it exists 38 | if "main" in dir(main) and callable(getattr(main, "main")): 39 | main.main() 40 | -------------------------------------------------------------------------------- /transformerlab/plugin_sdk/transformerlab/sdk/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/plugin_sdk/transformerlab/sdk/__init__.py -------------------------------------------------------------------------------- /transformerlab/plugin_sdk/transformerlab/sdk/v1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/plugin_sdk/transformerlab/sdk/v1/__init__.py -------------------------------------------------------------------------------- /transformerlab/plugin_sdk/transformerlab/sdk/v1/export.py: -------------------------------------------------------------------------------- 1 | import time 2 | import traceback 3 | 4 | from transformerlab.sdk.v1.tlab_plugin import TLabPlugin, DotDict 5 | 6 | class ExportTLabPlugin(TLabPlugin): 7 | """Enhanced Decorator class for TransformerLab exporter plugins""" 8 | 9 | def __init__(self): 10 | super().__init__() 11 | self.tlab_plugin_type = "exporter" 12 | 13 | self._parser.add_argument("--output_dir", default=None, type=str, help="Path to save the exported model") 14 | 15 | def _ensure_args_parsed(self): 16 | """Ensure arguments are parsed and convert self.params to a DotDict""" 17 | if not self._args_parsed: 18 | args, unknown_args = self._parser.parse_known_args() 19 | 20 | # Transfer all known arguments to attributes of self 21 | for key, value in vars(args).items(): 22 | self.params[key] = value 23 | 24 | self._parse_unknown_args(unknown_args) 25 | self._args_parsed = True 26 | 27 | if not isinstance(self.params, DotDict): 28 | self.params = DotDict(self.params) 29 | 30 | def _parse_unknown_args(self, unknown_args): 31 | """Parse unknown arguments which change with each export job""" 32 | key = None 33 | for arg in unknown_args: 34 | if arg.startswith("--"): # Argument key 35 | key = arg.lstrip("-") 36 | self.params[key] = True 37 | elif key: # Argument value 38 | self.params[key] = arg 39 | key = None 40 | 41 | # Added exporter-specific functionality and removed wandb logging 42 | def exporter_job_wrapper(self, progress_start: int = 0, progress_end: int = 100): 43 | """Decorator for wrapping an exporter function with job status updates""" 44 | 45 | def decorator(func): 46 | def wrapper(*args, **kwargs): 47 | # Ensure args are parsed and job is initialized 48 | self._ensure_args_parsed() 49 | start_time = time.strftime("%Y-%m-%d %H:%M:%S") 50 | self.add_job_data("start_time", start_time) 51 | self.add_job_data("model_name", self.params.model_name) 52 | 53 | #Update starting progress 54 | self.job.update_progress(progress_start) 55 | 56 | try: 57 | #Call the wrapped function 58 | result = func(*args, **kwargs) 59 | 60 | #Update final progress and success status 61 | self.job.update_progress(progress_end) 62 | self.job.set_job_completion_status("success", "Export completed successfully") 63 | self.add_job_data("end_time", time.strftime("%Y-%m-%d %H:%M:%S")) 64 | 65 | return result 66 | 67 | except Exception as e: 68 | # Capture the full erorr 69 | error_msg = f"Error in Job: {str(e)}\n{traceback.format_exc()}" 70 | print(error_msg) 71 | 72 | # Log the error 73 | self.job.set_job_completion_status("failed", f"Error occured: {str(e)}") 74 | self.add_job_data("end_time", time.strftime("%Y-%m-%d %H:%M:%S")) 75 | 76 | raise 77 | 78 | return wrapper 79 | 80 | return decorator 81 | 82 | 83 | # Create an instance of the ExportTLabPlugin class 84 | tlab_exporter = ExportTLabPlugin() -------------------------------------------------------------------------------- /transformerlab/plugins/airllm_mlx_server/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MLX Server - AirLLM", 3 | "uniqueId": "airllm_mlx_server", 4 | "description": "Performing inference using AirLLM on your Apple Silicon machine", 5 | "plugin-format": "python", 6 | "type": "loader", 7 | "version": "0.1.2", 8 | "model_architectures": [ 9 | "MLX", 10 | "CohereForCausalLM", 11 | "DeepseekV2ForCausalLM", 12 | "GemmaForCausalLM", 13 | "Gemma2ForCausalLM", 14 | "Gemma3ForCausalLM", 15 | "GPTBigCodeForCausalLM", 16 | "LlamaForCausalLM", 17 | "MistralForCausalLM", 18 | "MixtralForCausalLM", 19 | "PhiForCausalLM", 20 | "Phi3ForCausalLM", 21 | "Qwen2ForCausalLM" 22 | ], 23 | "supported_hardware_architectures": ["mlx"], 24 | "files": ["main.py", "setup.sh"], 25 | "setup-script": "setup.sh", 26 | "parameters": {} 27 | } 28 | -------------------------------------------------------------------------------- /transformerlab/plugins/airllm_mlx_server/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | uv pip install mlx==0.23.2 --upgrade 3 | uv pip install "mlx-lm==0.22.1" --upgrade 4 | uv pip install "mlx_embedding_models==0.0.11" 5 | uv pip install airllm==2.11.0 6 | -------------------------------------------------------------------------------- /transformerlab/plugins/autotrain_sft_trainer/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Autotrain SFT Trainer", 3 | "uniqueId": "autotrain_sft_trainer", 4 | "description": "SFT training using Huggingface autotrain", 5 | "plugin-format": "python", 6 | "type": "trainer", 7 | "version": "0.1.12", 8 | "model_architectures": [ 9 | "LlamaForCausalLM", 10 | "MistralForCausalLM", 11 | "MixtralForCausalLM", 12 | "PhiForCausalLM", 13 | "GemmaForCausalLM", 14 | "Qwen2ForCausalLM", 15 | "Qwen3ForCausalLM", 16 | "Qwen3MoeForCausalLM", 17 | "Phi3ForCausalLM" 18 | ], 19 | "supported_hardware_architectures": [], 20 | "files": ["main.py", "info.md", "setup.sh"], 21 | "setup-script": "setup.sh", 22 | "parameters": { 23 | "batch_size": { 24 | "title": "Batch Size", 25 | "type": "integer", 26 | "default": 4, 27 | "minimum": 1 28 | }, 29 | "learning_rate": { 30 | "title": "Learning Rate", 31 | "type": "number", 32 | "default": 2e-4, 33 | "minimum": 1e-6 34 | }, 35 | "num_train_epochs": { 36 | "title": "Number of Training Epochs", 37 | "type": "integer", 38 | "default": 1, 39 | "minimum": 1 40 | }, 41 | "adaptor_name": { 42 | "title": "Adaptor Name", 43 | "type": "string", 44 | "default": "adaptor", 45 | "required": true 46 | }, 47 | "log_to_wandb": { 48 | "title": "Log to Weights and Biases", 49 | "type": "boolean", 50 | "default": false, 51 | "required": true 52 | } 53 | }, 54 | "parameters_ui": { 55 | "batch_size": { 56 | "ui:help": "Default batch is 4. Setting this to 1 or 2 will reduce memory consumption but may slow performance." 57 | }, 58 | "learning_rate": { 59 | "ui:help": "Learning rate default is 2e-4." 60 | }, 61 | "log_to_wandb": { 62 | "ui:help": "Log training to Weights and Biases. You must have a Weights and Biases account and API key to use this feature. You need to set the API Key in settings to use this feature." 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /transformerlab/plugins/autotrain_sft_trainer/info.md: -------------------------------------------------------------------------------- 1 | # SFT Training 2 | (powered by Hugging Face Autotrain) 3 | 4 | ### Introduction 5 | 6 | In this plugin we implement RLHF using SFT (Supervised Fine-tuning). The plugin uses Hugging Face's autotrain-advanced library under the covers. 7 | 8 | ### Data format 9 | 10 | 11 | 12 | ### Links and Original Papers: 13 | 14 | - Hugging Face AutoTrain 15 | -------------------------------------------------------------------------------- /transformerlab/plugins/autotrain_sft_trainer/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install autotrain-advanced 2 | uv pip install bitsandbytes==0.45.5 --upgrade -------------------------------------------------------------------------------- /transformerlab/plugins/basic_evals/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Basic Evaluation Metrics", 3 | "uniqueId": "basic_evals", 4 | "evalsType": "dataset", 5 | "description": "Evaluating outputs of LLMs using basic defined metrics", 6 | "plugin-format": "python", 7 | "type": "evaluator", 8 | "version": "0.1.4", 9 | "git": "", 10 | "url": "", 11 | "files": ["main.py", "setup.sh"], 12 | "supported_hardware_architectures": ["cpu", "cuda", "mlx", "amd"], 13 | "_dataset": true, 14 | "_dataset_display_message": "Please make sure the name of the input and output columns match the ones in your dataset", 15 | "setup-script": "setup.sh", 16 | "parameters": { 17 | "predefined_tasks": { 18 | "type": "string", 19 | "title": "Pre-defined Evaluation Metrics", 20 | "enum": [ 21 | "Is Valid JSON", 22 | "Word Count", 23 | "Contains bulleted lists", 24 | "Contains headings", 25 | "Contains URLs", 26 | "Contains code blocks", 27 | "Contains tables", 28 | "Contains images", 29 | "Contains numbered lists", 30 | "Contains bold text", 31 | "Contains italic text", 32 | "Contains underline text", 33 | "Contains strikethrough text", 34 | "Contains blockquotes", 35 | "Contains inline code", 36 | "Contains emojis", 37 | "Contains email addresses", 38 | "Contains phone numbers", 39 | "Contains dates", 40 | "Contains times", 41 | "Contains numbers" 42 | ] 43 | }, 44 | "tasks": { 45 | "type": "array", 46 | "title": "Add Custom Evaluation Metrics", 47 | "items": { 48 | "type": "object", 49 | "properties": { 50 | "name": { 51 | "type": "string", 52 | "title": "Evaluation Name" 53 | }, 54 | "expression": { 55 | "type": "string", 56 | "title": "Regular Expression" 57 | }, 58 | "return_type": { 59 | "type": "string", 60 | "title": "Output Type", 61 | "enum": ["boolean", "number", "contains", "isequal", "code"] 62 | } 63 | } 64 | } 65 | }, 66 | "limit": { 67 | "title": "Fraction of samples to evaluate", 68 | "type": ["number"], 69 | "minimum": 0.0, 70 | "default": 1.0, 71 | "maximum": 1.0, 72 | "multipleOf": 0.1 73 | }, 74 | "input_col": { 75 | "type": "string", 76 | "title": "Input Column", 77 | "default": "input" 78 | }, 79 | "output_col": { 80 | "type": "string", 81 | "title": "Output Column", 82 | "default": "output" 83 | } 84 | }, 85 | "parameters_ui": { 86 | "limit": { 87 | "ui:help": "Enter a fraction of samples to evaluate from your data. Set as 1 to get all samples", 88 | "ui:widget": "RangeWidget" 89 | }, 90 | "predefined_tasks": { 91 | "ui:help": "Select an evaluation metric from the drop-down list", 92 | "ui:widget": "AutoCompleteWidget" 93 | }, 94 | "tasks": { 95 | "ui:help": "Set your own evaluation metric using regular expressions or set a specific string to match", 96 | "ui:widget": "EvaluationWidget" 97 | }, 98 | "input_col": { 99 | "ui:help": "Enter the name of the input column in your dataset" 100 | }, 101 | "output_col": { 102 | "ui:help": "Enter the name of the output column in your dataset" 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /transformerlab/plugins/basic_evals/info.md: -------------------------------------------------------------------------------- 1 | # Basic Evaluation Metrics 2 | 3 | ## Overview 4 | 5 | This plugin helps you evaluate Language Model (LLM) outputs using predefined metrics and custom evaluation criteria. You can use ready-made metrics or create your own using regular expressions. 6 | 7 | ## Features 8 | 9 | ### Pre-defined Metrics 10 | 11 | Choose from a variety of built-in evaluation metrics: 12 | 13 | - Content Structure (headings, lists, tables) 14 | - Text Formatting (bold, italic, underline) 15 | - Special Elements (code blocks, URLs, images) 16 | - Data Validation (JSON, numbers, dates) 17 | - Text Analysis (word count, emojis) 18 | 19 | ### Custom Evaluation 20 | 21 | Create your own metrics with: 22 | 23 | - Custom names for each metric 24 | - Regular expressions for pattern matching 25 | - Multiple return types: 26 | - boolean: Yes/No answers 27 | - number: Count occurrences 28 | - contains: Check if text contains a pattern 29 | - isequal: Exact match comparison 30 | - Code: Define custom Python code for evaluation (Must have a function called `evaluate` which runs all code) 31 | 32 | ## Getting Started 33 | 34 | ### 1. Prepare Your Dataset 35 | 36 | - Ensure your dataset has input and output columns 37 | - Default column names are "input" and "output" 38 | - You can customize column names if needed 39 | 40 | ### 2. Choose Evaluation Methods 41 | 42 | #### Using Pre-defined Metrics 43 | 44 | Simply select from the dropdown list of available metrics like: 45 | 46 | ```json 47 | [ 48 | "Is Valid JSON", 49 | "Word Count", 50 | "Contains URLs", 51 | "Contains code blocks" 52 | ] 53 | ``` 54 | 55 | #### Creating Custom Metrics 56 | 57 | Define your own evaluation tasks: 58 | 59 | ```json 60 | { 61 | "name": "Contains Numbers", 62 | "expression": "\\d+", 63 | "return_type": "boolean" 64 | } 65 | ``` 66 | 67 | > Note: Using the `Code` type requires a Python function called `evaluate` that runs the custom code. We only allow the safe, utility and limited builtins from [RestrictedPython](https://github.com/zopefoundation/RestrictedPython). Additionally we also support `re` and `json` modules but no other import is allowed within the code. The code should return a boolean value or a numeric value (int/float) only. No other return types are supported. 68 | 69 | ### 3. Configure Evaluation Settings 70 | 71 | - **Data Sample Size**: Choose how much of your dataset to evaluate (0.1 to 1.0) 72 | - **Column Names**: Specify input and output column names if different from defaults 73 | 74 | ### Example Usage 75 | 76 | Here's a complete evaluation setup example: 77 | 78 | ```json 79 | { 80 | "predefined_tasks": ["Word Count", "Contains URLs"], 81 | "tasks": [ 82 | { 83 | "name": "Has Date", 84 | "expression": "\\d{2}/\\d{2}/\\d{4}", 85 | "return_type": "boolean" 86 | } 87 | ], 88 | "limit": 0.5, 89 | "input_col": "prompt", 90 | "output_col": "response" 91 | } 92 | ``` 93 | 94 | This configuration will: 95 | 96 | - Use two predefined metrics 97 | - Add a custom date detection metric 98 | - Evaluate 50% of the dataset 99 | - Use "prompt" and "response" as column names -------------------------------------------------------------------------------- /transformerlab/plugins/basic_evals/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install pandas tensorboardX RestrictedPython==8.0.0 -------------------------------------------------------------------------------- /transformerlab/plugins/batched_generation_datasets/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Batched Output Generation from Datasets", 3 | "uniqueId": "batched_generation_datasets", 4 | "description": "Use a local or commercial LLM to generated outputs for the dataset generated.", 5 | "plugin-format": "python", 6 | "type": "generator", 7 | "version": "0.1.5", 8 | "git": "", 9 | "url": "", 10 | "files": ["main.py", "setup.sh"], 11 | "supported_hardware_architectures": ["cpu", "cuda", "mlx"], 12 | "_dataset": true, 13 | "_dataset_display_message": "Please upload a dataset file with columns: 'input', 'output', 'expected_output'. The context column is optional if using metrics which don't require it.", 14 | "setup-script": "setup.sh", 15 | "parameters": { 16 | "generation_model": { 17 | "title": "Generation Model (Model to be used for Generation. Select `local` to use the local model running)", 18 | "type": "string", 19 | "default": "Local" 20 | }, 21 | "system_prompt": { 22 | "title": "System Prompt", 23 | "type": "string" 24 | }, 25 | "input_column": { 26 | "title": "Input Column", 27 | "type": "string" 28 | }, 29 | "output_column": { 30 | "title": "Output Column", 31 | "type": "string" 32 | }, 33 | "batch_size": { 34 | "title": "Batch Size", 35 | "type": "integer", 36 | "default": 128 37 | }, 38 | "temperature": { 39 | "title": "Temperature", 40 | "type": "number", 41 | "default": 0.7, 42 | "minimum": 0.0, 43 | "maximum": 2.0, 44 | "multipleOf": 0.01 45 | }, 46 | "top_p": { 47 | "title": "Top P", 48 | "type": "number", 49 | "default": 1.0, 50 | "minimum": 0.0, 51 | "maximum": 1.0, 52 | "multipleOf": 0.1 53 | }, 54 | "max_tokens": { 55 | "title": "Max Tokens", 56 | "type": "integer", 57 | "default": 1024, 58 | "minimum": 1, 59 | "multipleOf": 1 60 | }, 61 | "dataset_split": { 62 | "title": "Dataset Split", 63 | "type": "string", 64 | "default": "train", 65 | "enum": ["train", "valid", "test"] 66 | } 67 | }, 68 | "parameters_ui": { 69 | "generation_model": { 70 | "ui:help": "Select the model to be used for generation from the drop-down list", 71 | "ui:widget": "ModelProviderWidget", 72 | "ui:options": { 73 | "multiple": false 74 | } 75 | }, 76 | "input_column": { 77 | "ui:help": "Select the column from the dataset to be used as input for generation" 78 | }, 79 | "output_column": { 80 | "ui:help": "Select the column from the dataset to be used as output for generation" 81 | }, 82 | "batch_size": { 83 | "ui:help": "Select the batch size for sending the data to the model for generation" 84 | }, 85 | "system_prompt": { 86 | "ui:help": "Enter the system prompt to be used for generation" 87 | }, 88 | "temperature": { 89 | "ui:help": "Select the temperature for generation", 90 | "ui:widget": "RangeWidget" 91 | }, 92 | "max_tokens": { 93 | "ui:help": "Select the maximum tokens for generation", 94 | "ui:widget": "RangeWidget" 95 | }, 96 | "dataset_split": { 97 | "ui:help": "Select the dataset split to be used for generation", 98 | "ui:widget": "AutoCompleteWidget", 99 | "ui:options": { 100 | "multiple": false 101 | } 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /transformerlab/plugins/batched_generation_datasets/info.md: -------------------------------------------------------------------------------- 1 | # Batched Output Generation from Datasets 2 | 3 | ## Overview 4 | 5 | The Batched Output Generation from Datasets plugin is designed to generate outputs for a synthetic dataset using either a local or a commercial LLM. This plugin processes your data in batches and allows you to customize how generation is performed for your data based on the selected model and parameters. 6 | 7 | ## Parameters 8 | 9 | ### generation_model 10 | 11 | - **Title:** Generation Model (Model to be used for Generation. Select `Local` to use the local model running) 12 | - **Type:** string 13 | - **Description:** Choose the model for output generation. Options include: 14 | - Local 15 | - Claude 3.5 Haiku 16 | - Claude 3.5 Sonnet 17 | - OpenAI GPT 4o 18 | - OpenAI GPT 4o Mini 19 | 20 | ### system_prompt 21 | 22 | - **Title:** System Prompt 23 | - **Type:** string 24 | - **Description:** Define the system prompt to guide the model during generation, if needed. 25 | 26 | ### input_column 27 | 28 | - **Title:** Input Column 29 | - **Type:** string 30 | - **Description:** Specify the column in your dataset that contains the input for generation. 31 | 32 | ### output_column 33 | 34 | - **Title:** Output Column 35 | - **Type:** string 36 | - **Description:** Specify the column where the generated outputs should be stored. 37 | 38 | ### batch_size 39 | 40 | - **Title:** Batch Size 41 | - **Type:** integer 42 | - **Default:** 128 43 | - **Description:** Set the number of records to process in each batch. 44 | 45 | ### temperature 46 | 47 | - **Title:** Temperature 48 | - **Type:** number 49 | - **Default:** 0.01 50 | - **Range:** 0.0 to 2.0 (increments of 0.01) 51 | - **Description:** Adjusts the temperature of the generation. 52 | 53 | ### top_p 54 | 55 | - **Title:** Top P 56 | - **Type:** number 57 | - **Default:** 1.0 58 | - **Range:** 0.0 to 1.0 (increments of 0.1) 59 | - **Description:** Controls the diversity of token selection during generation. 60 | 61 | ### max_tokens 62 | 63 | - **Title:** Max Tokens 64 | - **Type:** integer 65 | - **Default:** 1024 66 | - **Range:** 1 to 4096 67 | - **Description:** Sets the maximum number of tokens to be generated. 68 | 69 | ## Usage 70 | 71 | 1. **Configure the Plugin Parameters:** 72 | 73 | - Choose the appropriate generation model using the `generation_model` parameter. 74 | - Provide a custom `system_prompt` to guide the LLM’s behavior. 75 | - Specify the column names for both input and output using `input_column` and `output_column`. 76 | - Set the desired `batch_size`, `temperature`, `top_p`, and `max_tokens` to fine-tune the generation process. 77 | 78 | 2. **Run the Generation Process:** 79 | With the parameters set, execute the plugin script (`main.py`) or use the provided setup script (`setup.sh`) to generate outputs for your dataset. 80 | 81 | 3. **Review and Adjust:** 82 | Once the generation is complete, verify the results in your output column. If the outputs require adjustments, you can alter the parameters and run the process again. 83 | -------------------------------------------------------------------------------- /transformerlab/plugins/batched_generation_datasets/main.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from requests_batching import process_dataset 3 | 4 | from transformerlab.sdk.v1.generate import tlab_gen 5 | 6 | 7 | async def generate_batched(trlab_model, df: pd.DataFrame, sys_prompt_col=None) -> pd.DataFrame: 8 | updated_df = await process_dataset( 9 | df, 10 | batch_size=tlab_gen.params.batch_size, 11 | model=trlab_model.generation_model_name, 12 | inference_url=trlab_model.chat_completions_url, 13 | api_key=trlab_model.api_key, 14 | sys_prompt_col=sys_prompt_col, 15 | input_col=tlab_gen.params.input_column, 16 | output_col=tlab_gen.params.output_column, 17 | temperature=float(tlab_gen.params.temperature), 18 | max_tokens=int(tlab_gen.params.max_tokens), 19 | top_p=float(tlab_gen.params.top_p), 20 | ) 21 | return updated_df 22 | 23 | 24 | @tlab_gen.async_job_wrapper(progress_start=0, progress_end=100) 25 | async def run_generation(): 26 | """Main function for batched generation""" 27 | print(f"Generation type: {tlab_gen.params.generation_type}") 28 | print(f"Model Name: {tlab_gen.params.generation_model}") 29 | print(f"Dataset Name: {tlab_gen.params.dataset_name}") 30 | 31 | # Load the dataset ('train' split) using tlab_gen's built-in method 32 | dataset = tlab_gen.load_dataset([tlab_gen.params.dataset_split]) 33 | df = dataset[tlab_gen.params.dataset_split].to_pandas() 34 | print(f"Dataset loaded successfully with {len(df)} rows") 35 | tlab_gen.progress_update(20) 36 | 37 | # Apply system prompt if provided 38 | sys_prompt_col = None 39 | if tlab_gen.params.system_prompt: 40 | print(f"Using system prompt: {tlab_gen.params.system_prompt}") 41 | df["system_prompt"] = tlab_gen.params.system_prompt 42 | sys_prompt_col = "system_prompt" 43 | 44 | # Check if we're using a local model and verify the server is running 45 | if "local" in tlab_gen.params.generation_model.lower(): 46 | tlab_gen.check_local_server() 47 | 48 | # Load the model for generation 49 | trlab_model = tlab_gen.load_evaluation_model(field_name="generation_model") 50 | print("Model loaded successfully") 51 | tlab_gen.progress_update(30) 52 | 53 | # Run batched generation 54 | print("Running batched generation...") 55 | updated_df = await generate_batched(trlab_model, df, sys_prompt_col=sys_prompt_col) 56 | print("Batched generation completed successfully") 57 | tlab_gen.progress_update(80) 58 | 59 | # Save the results as a new dataset 60 | metadata = { 61 | "generation_method": "batched", 62 | "input_column": tlab_gen.params.input_column, 63 | "output_column": tlab_gen.params.output_column, 64 | "system_prompt": tlab_gen.params.get("system_prompt", None), 65 | "batch_size": tlab_gen.params.get("batch_size", 128), 66 | "temperature": tlab_gen.params.get("temperature", 0.01), 67 | "max_tokens": tlab_gen.params.get("max_tokens", 1024), 68 | "top_p": tlab_gen.params.get("top_p", 1.0), 69 | "source_dataset": tlab_gen.params.dataset_name, 70 | "dataset_split": tlab_gen.params.get("dataset_split", "train"), 71 | } 72 | 73 | output_file, dataset_name = tlab_gen.save_generated_dataset(updated_df, metadata) 74 | tlab_gen.progress_update(100) 75 | 76 | print(f"Dataset processed successfully as {dataset_name}") 77 | print(f"Saved to {output_file}") 78 | 79 | return updated_df 80 | 81 | 82 | run_generation() 83 | -------------------------------------------------------------------------------- /transformerlab/plugins/batched_generation_datasets/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install deepeval==2.8.2 langchain-openai==0.3.16 langchain==0.3.25 langchain-community==0.3.23 langchain_text_splitters==0.3.8 instructor==1.7.9 anthropic==0.49.0 datasets==3.5.1 2 | -------------------------------------------------------------------------------- /transformerlab/plugins/common-eleuther-ai-lm-eval-harness-mlx/info.md: -------------------------------------------------------------------------------- 1 | # Common EleutherAI LM Eval Harness (MLX) 2 | 3 | ## Overview 4 | 5 | The Eleuther AI LM Evaluation Harness plugin is designed to benchmark language models on some of the most commonly used tasks and all their variants available in the EleutherAI Harness suite. This plugin provides a comprehensive set of tasks to evaluate different aspects of language models. It is specially designed for the MLX platform, which allows you to run the evaluation harness on Mac devices. 6 | 7 | ## Supported Tasks 8 | 9 | - **MMLU (Massive Multitask Language Understanding):** 10 | This comprehensive benchmark is open access and widely used for evaluating models across various academic subjects. 11 | 12 | - **ARC (AI2 Reasoning Challenge):** 13 | Both ARC Easy and ARC Challenge are open access tasks that evaluate a model's ability to answer grade-school science questions. 14 | 15 | - **HellaSwag:** 16 | This open access task assesses common sense reasoning and situational understanding. 17 | 18 | - **WinoGrande:** 19 | An open access benchmark for testing commonsense reasoning and coreference resolution. 20 | 21 | - **PIQA (Physical Interaction Question Answering):** 22 | This open access task evaluates physical commonsense reasoning. 23 | 24 | - **BIG-Bench Hard (BBH):** 25 | A suite of 23 challenging tasks from the broader BIG-Bench collection, which is open access. 26 | 27 | ## Parameters 28 | 29 | ### task 30 | 31 | - **Title:** Task 32 | - **Type:** string 33 | - **Description:** Select the task you want to use for benchmarking. The available options cover a wide range of evaluation metrics related to reasoning, language understanding, etc. 34 | 35 | ### limit 36 | 37 | - **Title:** Limit 38 | - **Type:** number 39 | - **Description:** A fraction of samples to run for evaluation. Enter `1` or leave it as the default to use all samples. This parameter allows you to control the number of samples used for evaluation, which can be useful for quicker tests or limited resources. 40 | 41 | ## Usage 42 | 43 | 1. **Select the Task:** Choose the task that best fits your benchmarking needs from the `task` parameter. 44 | 2. **Set the Limit (if needed):** Enter a fraction of samples to run for evaluation in the `limit` parameter. Use `1` or leave it as the default to evaluate all samples. 45 | -------------------------------------------------------------------------------- /transformerlab/plugins/common-eleuther-ai-lm-eval-harness-mlx/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Check if the 'lm-evaluation-harness-mlx' directory exists and remove it if it does 4 | if [ -d "./lm-evaluation-harness-mlx" ]; then 5 | rm -rf ./lm-evaluation-harness-mlx 6 | fi 7 | 8 | # Clone the repository 9 | git clone https://github.com/chimezie/lm-evaluation-harness-mlx || { echo "Git clone failed or repository already exists"; exit 1; } 10 | 11 | # Navigate to the directory 12 | cd lm-evaluation-harness-mlx 13 | 14 | git checkout mlx 15 | 16 | # Install dependencies 17 | uv pip install -e . 18 | 19 | uv pip install pandas 20 | uv pip install mlx==0.23.2 --upgrade 21 | uv pip install "mlx-lm==0.22.1" --upgrade 22 | -------------------------------------------------------------------------------- /transformerlab/plugins/common-eleuther-ai-lm-eval-harness/info.md: -------------------------------------------------------------------------------- 1 | # Common EleutherAI LM Eval Harness 2 | 3 | ## Overview 4 | 5 | The Eleuther AI LM Evaluation Harness plugin is designed to benchmark language models on some of the most commonly used tasks and all their variants available in the EleutherAI Harness suite. This plugin provides a comprehensive set of tasks to evaluate different aspects of language models. 6 | 7 | ## Supported Tasks 8 | 9 | - **MMLU (Massive Multitask Language Understanding):** 10 | This comprehensive benchmark is open access and widely used for evaluating models across various academic subjects. 11 | 12 | - **ARC (AI2 Reasoning Challenge):** 13 | Both ARC Easy and ARC Challenge are open access tasks that evaluate a model's ability to answer grade-school science questions. 14 | 15 | - **HellaSwag:** 16 | This open access task assesses common sense reasoning and situational understanding. 17 | 18 | - **WinoGrande:** 19 | An open access benchmark for testing commonsense reasoning and coreference resolution. 20 | 21 | - **PIQA (Physical Interaction Question Answering):** 22 | This open access task evaluates physical commonsense reasoning. 23 | 24 | - **BIG-Bench Hard (BBH):** 25 | A suite of 23 challenging tasks from the broader BIG-Bench collection, which is open access. 26 | 27 | ## Parameters 28 | 29 | ### task 30 | 31 | - **Title:** Task 32 | - **Type:** string 33 | - **Description:** Select the task you want to use for benchmarking. The available options cover a wide range of evaluation metrics related to reasoning, language understanding, etc. 34 | 35 | ### limit 36 | 37 | - **Title:** Limit 38 | - **Type:** number 39 | - **Description:** A fraction of samples to run for evaluation. Enter `1` or leave it as the default to use all samples. This parameter allows you to control the number of samples used for evaluation, which can be useful for quicker tests or limited resources. 40 | 41 | ## Usage 42 | 43 | 1. **Select the Task:** Choose the task that best fits your benchmarking needs from the `task` parameter. 44 | 2. **Set the Limit (if needed):** Enter a fraction of samples to run for evaluation in the `limit` parameter. Use `1` or leave it as the default to evaluate all samples. -------------------------------------------------------------------------------- /transformerlab/plugins/common-eleuther-ai-lm-eval-harness/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # git clone https://github.com/EleutherAI/lm-evaluation-harness 3 | # cd lm-evaluation-harness 4 | # pip install -e . 5 | uv pip install lm-eval==0.4.7 6 | uv pip install "lm-eval[api]" 7 | uv pip install pandas 8 | -------------------------------------------------------------------------------- /transformerlab/plugins/deepeval_llm_judge/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install deepeval==2.8.2 langchain-openai instructor anthropic datasets seaborn tensorboardX -------------------------------------------------------------------------------- /transformerlab/plugins/deepeval_objective/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DeepEval Objective Metrics", 3 | "uniqueId": "deepeval_objective", 4 | "description": "Evaluating outputs of LLMs using objective metrics", 5 | "plugin-format": "python", 6 | "type": "evaluator", 7 | "evalsType": "dataset", 8 | "version": "0.2.8", 9 | "git": "https://github.com/confident-ai/deepeval", 10 | "url": "https://github.com/confident-ai/deepeval", 11 | "files": ["main.py", "setup.sh"], 12 | "supported_hardware_architectures": ["cpu", "cuda", "mlx", "amd"], 13 | "_dataset": true, 14 | "_dataset_display_message": "Please upload a dataset file with columns: 'input', 'output', 'expected_output'. The context column is optional if using metrics which don't require it.", 15 | "setup-script": "setup.sh", 16 | "parameters": { 17 | "tasks": { 18 | "title": "Evaluation Metric", 19 | "type": "string", 20 | "enum": [ 21 | "Rouge", 22 | "BLEU", 23 | "Exact Match", 24 | "Quasi Exact Match", 25 | "Quasi Contains", 26 | "BERT Score" 27 | ] 28 | }, 29 | "limit": { 30 | "title": "Fraction of samples to evaluate", 31 | "type": ["number"], 32 | "minimum": 0.0, 33 | "default": 1.0, 34 | "maximum": 1.0, 35 | "multipleOf": 0.1 36 | } 37 | }, 38 | "parameters_ui": { 39 | "metrics": { 40 | "ui:help": "Select an evaluation metric from the drop-down list" 41 | }, 42 | "limit": { 43 | "ui:help": "Enter a fraction of samples to evaluate from your data. Set as 1 to get all samples", 44 | "ui:widget": "RangeWidget" 45 | }, 46 | "tasks": { 47 | "ui:help": "Select an evaluation metric from the drop-down list", 48 | "ui:widget": "AutoCompleteWidget" 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /transformerlab/plugins/deepeval_objective/info.md: -------------------------------------------------------------------------------- 1 | # DeepEval Objective Metrics 2 | 3 | ## Overview 4 | 5 | The DeepEval Objective Metrics plugin is designed to evaluate the outputs of Large Language Models (LLMs) using objective metrics. This plugin provides a set of predefined metrics to assess various aspects of generated content. 6 | 7 | ## Dataset Requirements 8 | 9 | A local dataset uploaded to the dataset in Transformer Lab is required. The dataset file must be in CSV format and should compulsorily have the following columns: 10 | 11 | - `input` 12 | - `output` 13 | - `expected_output` 14 | 15 | ## Parameters 16 | 17 | ### Evaluation Metric 18 | 19 | - **Type:** string 20 | - **Description:** Select the evaluation metric you want to use. The available options are: 21 | - **Rouge:** Measures the overlap of n-grams between the generated output and the expected output. 22 | - **BLEU:** Evaluates the precision of n-grams in the generated output compared to the expected output. 23 | - **Exact Match:** Checks if the generated output exactly matches the expected output. 24 | - **Quasi Exact Match:** Allows for minor variations while checking for an exact match. 25 | - **Quasi Contains:** Checks if the generated output contains the expected output with minor variations. 26 | - **BERT Score:** Uses BERT embeddings to evaluate the similarity between the generated output and the expected output. 27 | 28 | ### Output Path 29 | 30 | - **Type:** string 31 | - **Description:** Provide the local path where the evaluation results should be saved. 32 | 33 | ## Usage 34 | 35 | 1. **Select the Evaluation Metric:** Choose the metric that best fits your evaluation needs from the `metrics` parameter. 36 | 2. **Provide the Output Path:** Enter the path where you want to save the evaluation results in the `output_path` parameter. 37 | 3. **Specify the Dataset:** Enter the dataset to be used for evaluation in the `dataset` parameter. 38 | -------------------------------------------------------------------------------- /transformerlab/plugins/deepeval_objective/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install deepeval==2.8.2 bert-score==0.3.13 datasets==3.5.1 seaborn==0.13.2 tensorboardx==2.6.2.2 -------------------------------------------------------------------------------- /transformerlab/plugins/dpo_orpo_simpo_trainer_llama_factory/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DPO / ORPO / SIMPO RLHF - Llama Factory", 3 | "uniqueId": "dpo_orpo_simpo_trainer_llama_factory", 4 | "description": "An implementation of several Preference Optimization methods using Llama Factory.", 5 | "plugin-format": "python", 6 | "type": "trainer", 7 | "version": "0.0.11", 8 | "model_architectures": [ 9 | "LlamaForCausalLM", 10 | "MistralForCausalLM", 11 | "MixtralForCausalLM", 12 | "PhiForCausalLM", 13 | "GemmaForCausalLM", 14 | "Qwen2ForCausalLM", 15 | "Qwen3ForCausalLM", 16 | "Qwen3MoeForCausalLM", 17 | "Phi3ForCausalLM" 18 | ], 19 | "supported_hardware_architectures": ["cuda", "amd"], 20 | "files": ["main.py", "setup.sh"], 21 | "setup-script": "setup.sh", 22 | "training_template_format": "none", 23 | "training_data_instructions": "Ensure that the dataset below has the following required columns: \"conversations\", \"chosen\", \"rejected\"", 24 | 25 | "parameters": { 26 | "pref_loss": { 27 | "title": "Task", 28 | "type": "string", 29 | "enum": ["dpo", "orpo", "simpo"], 30 | "required": true 31 | }, 32 | "learning_rate": { 33 | "title": "Learning Rate", 34 | "type": "number", 35 | "default": 0.001, 36 | "minimum": 1e-6 37 | }, 38 | "num_train_epochs": { 39 | "title": "Number of Training Epochs", 40 | "type": "integer", 41 | "default": 1, 42 | "minimum": 1 43 | }, 44 | "max_steps": { 45 | "title": "Max Steps (-1 means no limit)", 46 | "type": "integer", 47 | "default": -1 48 | }, 49 | "adaptor_name": { 50 | "title": "Adaptor Name", 51 | "type": "string", 52 | "required": true, 53 | "default": "dpo" 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /transformerlab/plugins/dpo_orpo_simpo_trainer_llama_factory/info.md: -------------------------------------------------------------------------------- 1 | # DPO / ORPO / SIMPO RLHF Training 2 | (powered by Llama Factory) 3 | 4 | ### Introduction 5 | 6 | In this plugin we implement RLHF using DPO (Direct Preference Optimization) / ORPO (Odds Ratio Preference Optimization) / SIMPO (Simple Preference Optimization). These methods allow for preference optimzation without the need for a reward model. The plugin uses Llama Factory under the covers. 7 | 8 | ### Data format 9 | 10 | Llama Factory requires data to be structured in a very specific format. It requires a better response in chosen column and a worse response in rejected column. 11 | 12 | 13 | ```json 14 | [ 15 | { 16 | "conversations": "conversation before response (required)", 17 | "chosen": "preferred answer (required)", 18 | "rejected": "rejected answer (required)" 19 | } 20 | ] 21 | ``` 22 | 23 | An example dataset is here: 24 | 25 | - Llama Factory 26 | 27 | 28 | 29 | 30 | ### What is Preference Optimization 31 | 32 | Direct Preference Optimization (DPO) has emerged as a promising alternative for aligning Large Language Models (LLMs) to human or AI preferences. Unlike traditional alignment methods, which are based on reinforcement learning, DPO recasts the alignment formulation as a simple loss function that can be optimised directly on a dataset of preferences. 33 | 34 | (from huggingface preference tuning) 35 | 36 | ### Links and Original Papers: 37 | 38 | - Llama Factory 39 | - DPO 40 | - SIMPO 41 | - ORPO: Monolithic Preference Optimization without Reference Model 42 | -------------------------------------------------------------------------------- /transformerlab/plugins/dpo_orpo_simpo_trainer_llama_factory/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install "trl>=0.8.2" 2 | rm -rf LLaMA-Factory 3 | git clone https://github.com/hiyouga/LLaMA-Factory.git 4 | cd LLaMA-Factory 5 | git checkout beec77a0898a39d94f41c23920415f5b4873a23a # this is a known good version 6 | uv pip install -e .[torch,metrics] -------------------------------------------------------------------------------- /transformerlab/plugins/eleuther-ai-lm-evaluation-harness-mlx/info.md: -------------------------------------------------------------------------------- 1 | # Eleuther AI LM Evaluation Harness (MLX) 2 | 3 | ## Overview 4 | 5 | The Eleuther AI LM Evaluation Harness plugin is designed to benchmark language models on various benchmarks available in the EleutherAI Harness suite. This plugin provides a comprehensive set of tasks to evaluate different aspects of language models. 6 | This plugin is specially designed for the MLX platform, which allows you to run the evaluation harness on Mac devices. 7 | 8 | ## Parameters 9 | 10 | ### task 11 | - **Title:** Task 12 | - **Type:** string 13 | - **Description:** Select the task you want to use for benchmarking. The available options cover a wide range of evaluation metrics related to advanced AI risk, coordination, corrigibility, self-awareness, and more. 14 | 15 | ### limit 16 | - **Title:** Limit 17 | - **Type:** number 18 | - **Description:** A fraction of samples to run for evaluation. Enter `1` or leave it as the default to use all samples. This parameter allows you to control the number of samples used for evaluation, which can be useful for quicker tests or limited resources. 19 | 20 | ## Usage 21 | 22 | 1. **Select the Task:** Choose the task that best fits your benchmarking needs from the `task` parameter. 23 | 2. **Set the Limit (if needed):** Enter a fraction of samples to run for evaluation in the `limit` parameter. Use `1` or leave it as the default to evaluate all samples. -------------------------------------------------------------------------------- /transformerlab/plugins/eleuther-ai-lm-evaluation-harness-mlx/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Check if the 'lm-evaluation-harness-mlx' directory exists and remove it if it does 4 | if [ -d "./lm-evaluation-harness-mlx" ]; then 5 | rm -rf ./lm-evaluation-harness-mlx 6 | fi 7 | 8 | # Clone the repository 9 | git clone https://github.com/chimezie/lm-evaluation-harness-mlx || { echo "Git clone failed or repository already exists"; exit 1; } 10 | 11 | # Navigate to the directory 12 | cd lm-evaluation-harness-mlx 13 | 14 | git checkout mlx 15 | 16 | # Install dependencies 17 | uv pip install -e . 18 | 19 | uv pip install pandas 20 | 21 | uv pip install mlx==0.23.2 --upgrade 22 | uv pip install "mlx-lm==0.22.1" --upgrade -------------------------------------------------------------------------------- /transformerlab/plugins/eleuther-ai-lm-evaluation-harness/info.md: -------------------------------------------------------------------------------- 1 | # Eleuther AI LM Evaluation Harness 2 | 3 | ## Overview 4 | 5 | The Eleuther AI LM Evaluation Harness plugin is designed to benchmark language models on various benchmarks available in the EleutherAI Harness suite. This plugin provides a comprehensive set of tasks to evaluate different aspects of language models. 6 | 7 | ## Parameters 8 | 9 | ### task 10 | - **Title:** Task 11 | - **Type:** string 12 | - **Description:** Select the task you want to use for benchmarking. The available options cover a wide range of evaluation metrics related to advanced AI risk, coordination, corrigibility, self-awareness, and more. 13 | 14 | ### limit 15 | - **Title:** Limit 16 | - **Type:** number 17 | - **Description:** A fraction of samples to run for evaluation. Enter `1` or leave it as the default to use all samples. This parameter allows you to control the number of samples used for evaluation, which can be useful for quicker tests or limited resources. 18 | 19 | ## Usage 20 | 21 | 1. **Select the Task:** Choose the task that best fits your benchmarking needs from the `task` parameter. 22 | 2. **Set the Limit (if needed):** Enter a fraction of samples to run for evaluation in the `limit` parameter. Use `1` or leave it as the default to evaluate all samples. 23 | -------------------------------------------------------------------------------- /transformerlab/plugins/eleuther-ai-lm-evaluation-harness/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # git clone https://github.com/EleutherAI/lm-evaluation-harness 3 | # cd lm-evaluation-harness 4 | # pip install -e . 5 | uv pip install lm-eval==0.4.7 6 | uv pip install "lm-eval[api]" 7 | uv pip install pandas 8 | -------------------------------------------------------------------------------- /transformerlab/plugins/embedding_model_trainer/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Install Python packages needed to train with Sentence Transformers v3: 4 | # uv pip install --upgrade \ 5 | # torch \ 6 | # transformers==4.41.2 \ 7 | # "sentence-transformers>=3" \ 8 | # datasets==2.19.1 \ 9 | # huggingface_hub 10 | 11 | # (Optional) Add any other libraries you need, e.g. wandb, to track experiments 12 | # uv pip install wandb 13 | uv pip install sentence-transformers -------------------------------------------------------------------------------- /transformerlab/plugins/fastchat_server/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Fastchat Server", 3 | "uniqueId": "fastchat_server", 4 | "description": "Fastchat loads models for inference using Huggingface Transformers for generation.", 5 | "plugin-format": "python", 6 | "type": "loader", 7 | "version": "1.0.16", 8 | "supports": [ 9 | "chat", 10 | "completion", 11 | "visualize_model", 12 | "model_layers", 13 | "rag", 14 | "tools", 15 | "template", 16 | "embeddings", 17 | "tokenize", 18 | "logprobs", 19 | "batched" 20 | ], 21 | "model_architectures": [ 22 | "CohereForCausalLM", 23 | "FalconForCausalLM", 24 | "GemmaForCausalLM", 25 | "Gemma2ForCausalLM", 26 | "GPTBigCodeForCausalLM", 27 | "GraniteForCausalLM", 28 | "LlamaForCausalLM", 29 | "MistralForCausalLM", 30 | "MixtralForCausalLM", 31 | "PhiForCausalLM", 32 | "Phi3ForCausalLM", 33 | "Qwen2ForCausalLM", 34 | "ExaoneForCausalLM", 35 | "T5ForConditionalGeneration", 36 | "Gemma3ForCausalLM", 37 | "Gemma3ForConditionalGeneration", 38 | "AprielForCausalLM", 39 | "GPTNeoXForCausalLM", 40 | "Qwen3ForCausalLM", 41 | "Qwen3MoeForCausalLM" 42 | ], 43 | "supported_hardware_architectures": ["cpu", "cuda", "mlx", "amd"], 44 | "files": ["main.py", "setup.sh"], 45 | "setup-script": "setup.sh", 46 | "parameters": { 47 | "gpu_ids": { 48 | "title": "GPU IDs to use for Inference. Leaving blank will use all available GPUs", 49 | "type": "string", 50 | "default": "" 51 | }, 52 | "load_compressed": { 53 | "title": "Load compressed model", 54 | "type": "string", 55 | "default": "None", 56 | "enum": ["None", "8-bit", "4-bit"] 57 | }, 58 | "model_dtype": { 59 | "title": "Select a specific data type for the model", 60 | "type": "string", 61 | "enum": ["auto", "float16", "bfloat16", "float32"] 62 | } 63 | }, 64 | "parameters_ui": { 65 | "gpu_ids": { 66 | "ui:help": "Specify a comma-separated list of GPU IDs to use for inference. The IDs for each GPU can be found in the Computer tab. For example: 0,1,2,3" 67 | }, 68 | "model_dtype": { 69 | "ui:help": "Select a specific data type for the model. This might help with older GPUs that do not support bfloat16" 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /transformerlab/plugins/fastchat_server/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Everything should be installed by default 3 | uv pip install "transformerlab-inference>=0.2.38" 4 | 5 | if ! command -v rocminfo &> /dev/null; then 6 | uv pip install bitsandbytes 7 | fi -------------------------------------------------------------------------------- /transformerlab/plugins/fastchat_vision_server/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Fastchat Multimodal Server", 3 | "uniqueId": "fastchat_vision_server", 4 | "description": "Fastchat loads vision models for inference using Huggingface Transformers for generation.", 5 | "plugin-format": "python", 6 | "type": "loader", 7 | "version": "1.0.4", 8 | "model_architectures": [ 9 | "LlavaForConditionalGeneration", 10 | "Mistral3ForConditionalGeneration" 11 | ], 12 | "supported_hardware_architectures": [ 13 | "cpu", 14 | "cuda", 15 | "mlx" 16 | ], 17 | "files": [ 18 | "main.py", 19 | "setup.sh" 20 | ], 21 | "setup-script": "setup.sh", 22 | "parameters": { 23 | "num_gpus": { 24 | "title": "Number of GPUs", 25 | "type": "integer" 26 | }, 27 | "eight_bit": { 28 | "title": "Enable 8-bit compression", 29 | "type": "boolean", 30 | "default": false 31 | } 32 | }, 33 | "parameters_ui": { 34 | "num_gpus": { 35 | "ui:emptyValue": "", 36 | "ui:help": "Used to spread models over multiple GPUs. Leave empty to use default." 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /transformerlab/plugins/fastchat_vision_server/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Everything should be installed by default -------------------------------------------------------------------------------- /transformerlab/plugins/generate_rag_outputs/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | uv pip install pandas -------------------------------------------------------------------------------- /transformerlab/plugins/gguf_exporter/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "GGUF Exporter", 3 | "uniqueId": "gguf_exporter", 4 | "description": "Exports the current model to GGUF format so it can be run on computers without a GPU.", 5 | "plugin-format": "python", 6 | "type": "exporter", 7 | "version": "0.2.3", 8 | "model_architectures": [ 9 | "CohereForCausalLM", 10 | "FalconForCausalLM", 11 | "LlamaForCausalLM", 12 | "GemmaForCausalLM", 13 | "Gemma2ForCausalLM", 14 | "GPTJForCausalLM", 15 | "MistralForCausalLM", 16 | "MixtralForCausalLM", 17 | "PhiForCausalLM", 18 | "Phi3ForCausalLM", 19 | "Qwen2ForCausalLM", 20 | "Qwen3ForCausalLM", 21 | "Qwen3MoeForCausalLM" 22 | ], 23 | "supported_hardware_architectures": ["cpu", "cuda", "mlx", "amd"], 24 | "export_architecture": "GGUF", 25 | "files": ["main.py", "setup.sh"], 26 | "setup-script": "setup.sh", 27 | "parameters": { 28 | "outtype": { 29 | "title": "Output Format", 30 | "type": "string", 31 | "default": "q8_0", 32 | "enum": ["q8_0", "f16", "f32"] 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /transformerlab/plugins/gguf_exporter/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Unfortunately, llama-cpp-python doesn't support conversion yet 3 | # pip install llama-cpp-python 4 | # So need to copy llama.cpp and run from there 5 | #!/usr/bin/env bash 6 | # Unfortunately, llama-cpp-python doesn't support conversion yet 7 | # pip install llama-cpp-python 8 | # So need to copy llama.cpp and run from there 9 | git clone https://github.com/ggerganov/llama.cpp/ 10 | cd llama.cpp 11 | git checkout tags/b5219 # check out the tag called "b5219" which is the latest release april 29 2025 -------------------------------------------------------------------------------- /transformerlab/plugins/grpo_trainer_multi_gpu/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #pip install "datasets==2.9.0" "accelerate==0.21.0" "evaluate==0.4.0" loralib 3 | uv pip install trl bitsandbytes accelerate 4 | #pip install rouge-score tensorboard py7zr 5 | -------------------------------------------------------------------------------- /transformerlab/plugins/inference_evals/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install deepeval==2.8.2 langchain-openai==0.3.16 langchain==0.3.25 langchain-community==0.3.23 langchain_text_splitters==0.3.8 instructor==1.7.9 anthropic==0.49.0 datasets==3.5.1 tensorboardx==2.6.2.2 -------------------------------------------------------------------------------- /transformerlab/plugins/llama_cpp_server/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Llama-cpp Server", 3 | "uniqueId": "llama_cpp_server", 4 | "description": "Runs llama-cpp-python server that can run GGUF models that work well on CPU only machines.", 5 | "plugin-format": "python", 6 | "type": "loader", 7 | "version": "0.1.10", 8 | "model_architectures": ["GGUF"], 9 | "supported_hardware_architectures": ["cpu", "cuda", "mlx", "amd"], 10 | "files": ["main.py", "setup.sh"], 11 | "setup-script": "setup.sh", 12 | "parameters": { 13 | "n_gpu_layers": { 14 | "title": "Number of Models Layers to Offload to GPU (-1 for all, 'auto' for automatic selection)", 15 | "type": "string", 16 | "default": "auto" 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /transformerlab/plugins/llama_cpp_server/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # If we install llama-cpp-python[server] it will install 3 | # Pydantic2 which will break FastChat which depends on Pydantic1 4 | # So we will install llama-cpp-python only and implement our 5 | # own server using FastAPI 6 | 7 | echo "Setting up llama-cpp-python..." 8 | 9 | # Detect OS 10 | if [[ "$(uname)" == "Darwin" ]]; then 11 | # macOS - check for Metal support 12 | if [[ "$(uname -m)" == "arm64" || "$(sysctl -n machdep.cpu.brand_string)" == *"Apple"* ]]; then 13 | echo "Detected Mac with Apple Silicon - installing with Metal support" 14 | CMAKE_ARGS="-DGGML_METAL=on" uv pip install llama-cpp-python --upgrade --force-reinstall --no-cache-dir 15 | else 16 | echo "Detected Mac with Intel CPU - installing with OpenBLAS support" 17 | CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" uv pip install llama-cpp-python --upgrade --force-reinstall --no-cache-dir 18 | fi 19 | elif command -v nvidia-smi &> /dev/null; then 20 | # Linux/Other with CUDA detected 21 | echo "CUDA GPU detected. Installing based on CUDA setup using GGML CUDA" 22 | CMAKE_ARGS="-DGGML_CUDA=on" FORCE_CMAKE=1 uv pip install llama-cpp-python --force-reinstall --no-cache-dir 23 | 24 | elif command -v rocminfo &> /dev/null; then 25 | # Linux/Other with CUDA detected 26 | echo "AMD GPU detected. Installing based on AMD setup using GGML HIPBLAS" 27 | CMAKE_ARGS="-DGGML_HIPBLAS=on" FORCE_CMAKE=1 uv pip install llama-cpp-python --force-reinstall --no-cache-dir 28 | 29 | else 30 | # Linux/Other without CUDA - try using OpenBLAS 31 | echo "No GPU detected - installing with OpenBLAS support" 32 | uv pip install llama-cpp-python --upgrade --no-cache-dir --force-reinstall 33 | fi 34 | 35 | echo "llama-cpp-python installation complete." 36 | -------------------------------------------------------------------------------- /transformerlab/plugins/llama_trainer/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #pip install "datasets==2.9.0" "accelerate==0.21.0" "evaluate==0.4.0" loralib 3 | uv pip install trl 4 | # if we're NOT on AMD/ROCm, install bitsandbytes for quantization support 5 | if ! command -v rocminfo &> /dev/null; then 6 | uv pip install bitsandbytes 7 | fi 8 | -------------------------------------------------------------------------------- /transformerlab/plugins/llama_trainer_multi_gpu/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install trl bitsandbytes accelerate peft 2 | -------------------------------------------------------------------------------- /transformerlab/plugins/llamafile_exporter/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Llamafile Exporter", 3 | "uniqueId": "llamafile_exporter", 4 | "description": "Exports the current model to a fully contained self-executing llamafile.", 5 | "plugin-format": "python", 6 | "type": "exporter", 7 | "version": "0.1.5", 8 | "model_architectures": ["GGUF"], 9 | "supported_hardware_architectures": ["cpu", "cuda", "mlx", "amd"], 10 | "export_architecture": "llamafile", 11 | "files": ["main.py", "setup.sh"], 12 | "setup-script": "setup.sh", 13 | "parameters": {} 14 | } 15 | -------------------------------------------------------------------------------- /transformerlab/plugins/llamafile_exporter/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import shutil 4 | 5 | try: 6 | from transformerlab.sdk.v1.export import tlab_exporter 7 | except ImportError or ModuleNotFoundError: 8 | from transformerlab.plugin_sdk.transformerlab.sdk.v1.export import tlab_exporter 9 | 10 | 11 | tlab_exporter.add_argument( 12 | "--model_path", default="gpt-j-6b", type=str, help="Path to directory or file containing the model.") 13 | 14 | 15 | @tlab_exporter.exporter_job_wrapper(progress_start=0, progress_end=100) 16 | def llamafile_export(): 17 | """Export a model to Llamafile format""" 18 | input_model = tlab_exporter.params.get("model_name") 19 | input_model_id_without_author = input_model.split("/")[-1] 20 | input_model_path = tlab_exporter.params.get("model_path") 21 | output_dir = tlab_exporter.params.get("output_dir") # Must be created by plugin 22 | plugin_dir = os.path.realpath(os.path.dirname(__file__)) 23 | outfile_name = f"{input_model_id_without_author}.llamafile" 24 | argsfile = os.path.join(plugin_dir, ".args") 25 | 26 | # Create output file 27 | os.makedirs(output_dir, exist_ok=True) 28 | 29 | print("Starting Llamafile conversion...") 30 | tlab_exporter.progress_update(15) 31 | tlab_exporter.add_job_data("status", "Starting Llamafile conversion") 32 | print("Creating .args file...") 33 | 34 | argsoutput = f"""-m 35 | {input_model_id_without_author} 36 | --host 37 | 0.0.0.0 38 | -ngl 39 | 9999 40 | """ 41 | 42 | with open(argsfile, "w") as f: 43 | f.write(argsoutput) 44 | 45 | tlab_exporter.progress_update(30) 46 | tlab_exporter.add_job_data("status", "Creating base llamafile") 47 | print("Copying base llamafile...") 48 | 49 | base_llamafile = os.path.join(plugin_dir, "llamafile") 50 | temp_llamafile = os.path.join(plugin_dir, outfile_name) 51 | shutil.copy(base_llamafile, temp_llamafile) 52 | 53 | tlab_exporter.progress_update(50) 54 | tlab_exporter.add_job_data("status", "Merging model with Llamafile") 55 | 56 | # Merge files together in single executable using zipalign 57 | subprocess_cmd = ["sh", "./zipalign", "-j0", outfile_name, input_model_path, ".args"] 58 | export_process = subprocess.run( 59 | subprocess_cmd, cwd=plugin_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True 60 | ) 61 | 62 | stdout = export_process.stdout 63 | for line in stdout.strip().splitlines(): 64 | print(line) 65 | 66 | if export_process.returncode != 0: 67 | tlab_exporter.add_job_data("stderr", stdout) 68 | print(f"zipalign failed with code {export_process.returncode}") 69 | raise RuntimeError(f"zipalign failed with return code {export_process.returncode}") 70 | 71 | tlab_exporter.progress_update(80) 72 | tlab_exporter.add_job_data("status", "Moving Llamafile to output directory") 73 | print(f"Moving {outfile_name} to output directory {output_dir}...") 74 | 75 | final_llamafile_path = os.path.join(output_dir, outfile_name) 76 | shutil.move(temp_llamafile, final_llamafile_path) 77 | 78 | tlab_exporter.progress_update(100) 79 | tlab_exporter.add_job_data("status", "Llamafile creation complete") 80 | print("Llamafile export completed successfully!") 81 | 82 | return "Successful export to Llamafile format" 83 | 84 | 85 | llamafile_export() 86 | -------------------------------------------------------------------------------- /transformerlab/plugins/llamafile_exporter/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # TODO: Get latest version dynamically? 4 | LATEST_VERSION="0.9.0" 5 | 6 | # Download latest built of llamafile 7 | # This is possibly not great because this is 350MB. 8 | # But llamafile is over half of that 9 | curl -L https://github.com/Mozilla-Ocho/llamafile/releases/download/$LATEST_VERSION/llamafile-$LATEST_VERSION -o llamafile 10 | curl -L https://github.com/Mozilla-Ocho/llamafile/releases/download/$LATEST_VERSION/zipalign-$LATEST_VERSION -o zipalign 11 | 12 | # Set llamafile to be executable 13 | chmod +x llamafile 14 | chmod +x zipalign -------------------------------------------------------------------------------- /transformerlab/plugins/llamaindex_simple_document_search/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | uv pip install llama-index==0.12.38 3 | uv pip install llama-index-llms-openai-like==0.4.0 4 | uv pip install openai==1.82.1 5 | uv pip install llama-index-embeddings-huggingface==0.5.4 6 | uv pip install cryptography==44.0.2 # needed to read PDFs 7 | # uv pip install xformers 8 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_exporter/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MLX Exporter", 3 | "uniqueId": "mlx_exporter", 4 | "description": "Exports the current model to MLX format so it can be run on Apple Silicon.", 5 | "plugin-format": "python", 6 | "type": "exporter", 7 | "version": "1.0.18", 8 | "model_architectures": [ 9 | "CohereForCausalLM", 10 | "DeepseekV2ForCausalLM", 11 | "GemmaForCausalLM", 12 | "Gemma2ForCausalLM", 13 | "Gemma3ForCausalLM", 14 | "GPTBigCodeForCausalLM", 15 | "LlamaForCausalLM", 16 | "MistralForCausalLM", 17 | "MixtralForCausalLM", 18 | "PhiForCausalLM", 19 | "Phi3ForCausalLM", 20 | "Qwen2ForCausalLM", 21 | "Qwen3ForCausalLM", 22 | "Qwen3MoeForCausalLM" 23 | ], 24 | "export_architecture": "MLX", 25 | "files": ["main.py", "setup.sh"], 26 | "supported_hardware_architectures": ["mlx"], 27 | "setup-script": "setup.sh", 28 | "parameters": { 29 | "q_bits": { 30 | "title": "Bits per weight for quantization", 31 | "type": "string", 32 | "default": "4", 33 | "enum": ["8", "4", "2"] 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_exporter/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | uv pip install mlx==0.25.2 --upgrade 3 | uv pip install "mlx-lm==0.24.0" --upgrade 4 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_lora_trainer/info.md: -------------------------------------------------------------------------------- 1 | # Fine-Tuning with LoRA or QLoRA using Apple MLX 2 | 3 | ### Introduction 4 | 5 | If you have an Apple M-Series Chip, MLX LoRA fine-tuning is the fastest and simplest way to train models on your local computer. 6 | 7 | This implementation works with the following model families: 8 | 9 | - Mistral 10 | - Llama 11 | - Phi2 12 | - Mixtral 13 | - Qwen2 14 | - Gemma 15 | - OLMo 16 | - MiniCPM 17 | - InternLM2 18 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_lora_trainer/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | uv pip install trl 3 | uv pip install mlx==0.25.2 --upgrade 4 | uv pip install "mlx-lm==0.24.0" --upgrade 5 | # requires: 6 | # mlx>=0.0.7 7 | # transformers 8 | # numpya -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Apple MLX RLAIF PPO Trainer", 3 | "uniqueId": "mlx_rlaif_trainer", 4 | "description": "MLX PPO (Proximal Policy Optimization) Reinforcement Learning from AI Feedback (RLAIF) trainer for MLX models.", 5 | "plugin-format": "python", 6 | "type": "trainer", 7 | "version": "0.1.2", 8 | "model_architectures": [ 9 | "MLX", 10 | "LlamaForCausalLM", 11 | "MistralForCausalLM", 12 | "MixtralForCausalLM", 13 | "PhiForCausalLM", 14 | "Qwen2ForCausalLM" 15 | ], 16 | "training_template_format": "none", 17 | "training_data_instructions": "Enxure that the dataset below has the following required columns: \"conversations\", \"chosen\", \"rejected\"", 18 | "supported_hardware_architectures": ["mlx"], 19 | "files": ["main.py", "setup.sh"], 20 | "setup-script": "setup.sh", 21 | "parameters": { 22 | "batch_size": { 23 | "title": "Batch Size", 24 | "type": "integer", 25 | "default": 8, 26 | "minimum": 1 27 | }, 28 | "mini_batch_size": { 29 | "title": "Mini Batch Size", 30 | "type": "integer", 31 | "default": 8, 32 | "minimum": 1 33 | }, 34 | "ppo_epochs": { 35 | "title": "PPO Epochs", 36 | "type": "integer", 37 | "default": 4, 38 | "minimum": 1 39 | }, 40 | "num_steps": { 41 | "title": "Number of Steps", 42 | "type": "integer", 43 | "default": 5550, 44 | "minimum": 1 45 | }, 46 | "max_completion_length": { 47 | "title": "Max Completion Length", 48 | "type": "integer", 49 | "default": 256, 50 | "minimum": 1 51 | }, 52 | "init_kl_coef": { 53 | "title": "Initial KL Coefficient", 54 | "type": "number", 55 | "default": 0.2, 56 | "minimum": 0.0, 57 | "maximum": 10.0 58 | }, 59 | "seed": { 60 | "title": "Random Seed", 61 | "type": "integer", 62 | "default": 42, 63 | "minimum": 0 64 | }, 65 | "adaptor_name": { 66 | "title": "Adaptor Name", 67 | "type": "string", 68 | "default": "ppo", 69 | "required": true 70 | }, 71 | "ground_truth_reward": { 72 | "title": "Use Ground Truth Reward", 73 | "type": "boolean", 74 | "default": true 75 | }, 76 | "adap_kl_ctrl": { 77 | "title": "Adaptive KL Control", 78 | "type": "boolean", 79 | "default": true 80 | } 81 | }, 82 | "parameters_ui": { 83 | "batch_size": { 84 | "ui:help": "Batch size for PPO training." 85 | }, 86 | "mini_batch_size": { 87 | "ui:help": "Mini batch size for PPO updates." 88 | }, 89 | "ppo_epochs": { 90 | "ui:help": "Number of PPO epochs per update." 91 | }, 92 | "num_steps": { 93 | "ui:help": "Total number of PPO steps to run." 94 | }, 95 | "init_kl_coef": { 96 | "ui:help": "Initial KL divergence coefficient for PPO." 97 | }, 98 | "seed": { 99 | "ui:help": "Random seed for reproducibility." 100 | }, 101 | "adaptor_name": { 102 | "ui:help": "Name for the PPO-trained model/adaptor." 103 | }, 104 | "ground_truth_reward": { 105 | "ui:help": "Use ground truth reward for PPO training." 106 | }, 107 | "adap_kl_ctrl": { 108 | "ui:help": "Enable adaptive KL control during PPO." 109 | }, 110 | "max_completion_length": { 111 | "ui:help": "Maximum length of the completion for each response during PPO training." 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 andrew-silva 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/data/__init__.py -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/imessage_bot.md: -------------------------------------------------------------------------------- 1 | In this example, we'll try to build a chatbot to emulate ourselves based on iMessage history. 2 | This example is a work-in-progress, feel free to help improve the efficiency or performance of these scripts! 3 | 4 | ## Contents 5 | 6 | * [Getting the Data](#Data) 7 | * [Supervised Fine-tuning (SFT)](#Supervised-Fine-tuning) 8 | * [Learning a Reward Model](#Learning-a-Reward-Model) 9 | * [Reinforcement Learning (RLHF)](#Reinforcement-Learning-RLHF) 10 | * [Talk to your model](#Generate) 11 | 12 | 13 | ## Data 14 | 15 | I used the [imessage-exporter](https://github.com/ReagentX/imessage-exporter) to dump all of my iMessages into one directory. 16 | Download/install the tool, open up a Terminal, and run: 17 | ```bash 18 | imessage-exporter -f txt -o message_data 19 | ```` 20 | To export your messages in the `txt` format in an output directory called `message_data`. 21 | 22 | _**WARNING**_ -- _Do not share this data._ 23 | This should go without saying, but your iMessage data is extremely personal. 24 | Your shared family passwords, awkward inappropriate messages, and so much more are in this data. 25 | Please be careful with it. 26 | And keep in mind that your friends/family have likely not agreed to be in a dataset, so _do not share this data_. 27 | 28 | ## Supervised Fine-Tuning 29 | 30 | We'll start with a [meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf) model -- 31 | 32 | ``` 33 | python sft.py --model meta-llama/Llama-2-7b-chat-hf \ 34 | --train \ 35 | -- data /Users/path/to/your/message_data/ \ 36 | --iters 20000 \ 37 | --batch-size 1 \ 38 | --save-file lora_weights.npz 39 | ``` 40 | 41 | I have currently hard-coded the sequence length to be 256 in the `data_utils.py` script (Line 65-70). 42 | This means that memory usage is pretty high, and training is very slow. 43 | Open to-dos for (1) making this more flexible (config parameter for max-sequence-length) and (2) making this more memory efficient (QLoRA or other approach). 44 | 45 | The `sft.py` script will spit out an adapter file, and you should use `fuse.py` to turn that into a model to load in. 46 | 47 | ### Learning a Reward Model 48 | 49 | I haven't yet finished this piece of the puzzle! 50 | The plan is to generate synthetic data for learning a reward model using prompts from the message data. 51 | So when a friend sends me a message, the bad example is whatever Llama-2 responds and the good example is whatever I responded. 52 | 53 | ### Reinforcement Learning (RLHF) 54 | 55 | To run update your model to maximize scores under a 56 | ground truth or learned reward model, we will use the `ppo_training.py` script. 57 | 58 | ``` 59 | python ppo_training.py --model ./imessage_sft_fine_tune/ \ 60 | --reward_model /imessage_reward_model/ \ 61 | --log_with=wandb 62 | ``` 63 | 64 | This loads in your iMessage fine-tune with `--model ./imessage_sft_fine_tune`, which you trained above. 65 | 66 | ### Generate 67 | 68 | To chat with a trained model, use the `talk_to_model.py` script: 69 | 70 | ``` 71 | python talk_to_model.py --model ./imessage_sft_fine_tune/ \ 72 | --max-tokens 50 \ 73 | --temp 0.0 74 | ``` 75 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/models/__init__.py -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/models/convert.py: -------------------------------------------------------------------------------- 1 | # File copied from mlx-examples at https://github.com/ml-explore/mlx-examples/blob/main/lora/convert.py 2 | # 3 | # Copyright © 2023 Apple Inc. 4 | 5 | import argparse 6 | import copy 7 | 8 | import mlx.core as mx 9 | import mlx.nn as nn 10 | import utils 11 | from mlx.utils import tree_flatten 12 | 13 | 14 | def quantize(weights, config, args): 15 | quantized_config = copy.deepcopy(config) 16 | 17 | # Get model classes 18 | model_class, model_args_class = utils._get_classes(config=config) 19 | 20 | # Load the model: 21 | model = model_class(model_args_class.from_dict(config)) 22 | model.load_weights(list(weights.items()), strict=False) 23 | 24 | # Quantize the model: 25 | nn.QuantizedLinear.quantize_module( 26 | model, 27 | args.q_group_size, 28 | args.q_bits, 29 | linear_class_predicate=lambda m: isinstance(m, nn.Linear) 30 | and m.weight.shape[0] != 8, 31 | ) 32 | 33 | # Update the config: 34 | quantized_config["quantization"] = { 35 | "group_size": args.q_group_size, 36 | "bits": args.q_bits, 37 | } 38 | quantized_weights = dict(tree_flatten(model.parameters())) 39 | 40 | return quantized_weights, quantized_config 41 | 42 | 43 | if __name__ == "__main__": 44 | parser = argparse.ArgumentParser( 45 | description="Convert Hugging Face model to MLX format" 46 | ) 47 | parser.add_argument( 48 | "--hf-path", 49 | type=str, 50 | help="Path to the Hugging Face model.", 51 | ) 52 | parser.add_argument( 53 | "--mlx-path", 54 | type=str, 55 | default="mlx_model", 56 | help="Path to save the MLX model.", 57 | ) 58 | parser.add_argument( 59 | "-q", 60 | "--quantize", 61 | help="Generate a quantized model.", 62 | action="store_true", 63 | ) 64 | parser.add_argument( 65 | "--q-group-size", 66 | help="Group size for quantization.", 67 | type=int, 68 | default=64, 69 | ) 70 | parser.add_argument( 71 | "--q-bits", 72 | help="Bits per weight for quantization.", 73 | type=int, 74 | default=4, 75 | ) 76 | parser.add_argument( 77 | "--dtype", 78 | help="Type to save the parameters, ignored if -q is given.", 79 | type=str, 80 | choices=["float16", "bfloat16", "float32"], 81 | default="float16", 82 | ) 83 | parser.add_argument( 84 | "--upload-name", 85 | help="The name of model to upload to Hugging Face MLX Community", 86 | type=str, 87 | default=None, 88 | ) 89 | 90 | args = parser.parse_args() 91 | 92 | print("[INFO] Loading") 93 | weights, config, tokenizer = utils.fetch_from_hub(args.hf_path) 94 | 95 | dtype = mx.float16 if args.quantize else getattr(mx, args.dtype) 96 | weights = {k: v.astype(dtype) for k, v in weights.items()} 97 | if args.quantize: 98 | print("[INFO] Quantizing") 99 | weights, config = quantize(weights, config, args) 100 | 101 | utils.save_model(args.mlx_path, weights, tokenizer, config) 102 | if args.upload_name is not None: 103 | utils.upload_to_hub(args.mlx_path, args.upload_name, args.hf_path) 104 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/models/lora.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import mlx.core as mx 4 | import mlx.nn as nn 5 | 6 | 7 | class LoRALinear(nn.Module): 8 | @staticmethod 9 | def from_linear(linear: nn.Linear, rank: int = 8): 10 | # TODO remove when input_dims and output_dims are attributes 11 | # on linear and quantized linear 12 | output_dims, input_dims = linear.weight.shape 13 | if isinstance(linear, nn.QuantizedLinear): 14 | input_dims *= 32 // linear.bits 15 | lora_lin = LoRALinear(input_dims, output_dims, rank) 16 | lora_lin.linear = linear 17 | return lora_lin 18 | 19 | def to_linear(self, de_quantize: bool = False): 20 | linear = self.linear 21 | bias = "bias" in linear 22 | weight = linear.weight 23 | is_quantized = isinstance(linear, nn.QuantizedLinear) 24 | 25 | # Use the same type as the linear weight if not quantized 26 | dtype = weight.dtype 27 | 28 | if is_quantized: 29 | dtype = mx.float16 30 | weight = mx.dequantize( 31 | weight, 32 | linear.scales, 33 | linear.biases, 34 | linear.group_size, 35 | linear.bits, 36 | ) 37 | output_dims, input_dims = weight.shape 38 | fused_linear = nn.Linear(input_dims, output_dims, bias=bias) 39 | 40 | lora_b = (self.scale * self.lora_b.T).astype(dtype) 41 | lora_a = self.lora_a.T.astype(dtype) 42 | fused_linear.weight = weight + lora_b @ lora_a 43 | if bias: 44 | fused_linear.bias = linear.bias 45 | 46 | if is_quantized and not de_quantize: 47 | fused_linear = nn.QuantizedLinear.from_linear( 48 | fused_linear, 49 | linear.group_size, 50 | linear.bits, 51 | ) 52 | 53 | return fused_linear 54 | 55 | def __init__( 56 | self, 57 | input_dims: int, 58 | output_dims: int, 59 | lora_rank: int = 8, 60 | bias: bool = False, 61 | scale: float = 20.0, 62 | ): 63 | super().__init__() 64 | 65 | # Regular linear layer weights 66 | self.linear = nn.Linear(input_dims, output_dims, bias=bias) 67 | 68 | # Scale for low-rank update 69 | self.scale = scale 70 | 71 | # Low rank lora weights 72 | scale = 1 / math.sqrt(input_dims) 73 | self.lora_a = mx.random.uniform( 74 | low=-scale, 75 | high=scale, 76 | shape=(input_dims, lora_rank), 77 | ) 78 | self.lora_b = mx.zeros(shape=(lora_rank, output_dims)) 79 | 80 | def __call__(self, x): 81 | dtype = self.linear.weight.dtype 82 | if isinstance(self.linear, nn.QuantizedLinear): 83 | dtype = self.linear.scales.dtype 84 | y = self.linear(x.astype(dtype)) 85 | z = (x @ self.lora_a) @ self.lora_b 86 | return y + self.scale * z 87 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/models/prompt_tuning.py: -------------------------------------------------------------------------------- 1 | import mlx.core as mx 2 | import mlx.nn as nn 3 | 4 | 5 | class PromptTuning(nn.Module): 6 | def __init__( 7 | self, 8 | num_tokens: int, 9 | model: nn.Module 10 | ): 11 | super().__init__() 12 | 13 | # Regular linear layer weights 14 | self.model = model 15 | self.prompts = nn.Embedding(num_tokens, self.model.model.args.hidden_size) 16 | self.prompt_size = num_tokens 17 | 18 | def __call__(self, input_ids, attention_mask=None, cache=None): 19 | # dtype = self.prompts.weight.dtype 20 | # if isinstance(self.prompts, nn.QuantizedLinear): 21 | # dtype = self.prompts.scales.dtype 22 | if cache is None: 23 | prompt_embeds = mx.reshape(self.prompts(mx.arange(self.prompt_size)), 24 | (1, self.prompt_size, -1)) 25 | prompt_embeds = mx.repeat(prompt_embeds, input_ids.shape[0], axis=0) 26 | input_embeds = self.model.model.embed_tokens(input_ids) 27 | input_embeds = mx.concatenate([prompt_embeds, input_embeds], axis=1) 28 | if attention_mask is None: 29 | attention_mask = nn.MultiHeadAttention.create_additive_causal_mask(input_embeds.shape[1]) 30 | attention_mask[:, :self.prompt_size] = 0 31 | else: 32 | attention_mask = 1-attention_mask 33 | attention_mask *= -1e9 34 | if attention_mask.shape[-1] < input_embeds.shape[-1]: 35 | prompt_attn = mx.zeros((input_embeds.shape[0], input_embeds.shape[1]-attention_mask.shape[1])) 36 | attention_mask = mx.concatenate([prompt_attn, attention_mask], axis=1) 37 | attention_mask = attention_mask.astype(input_embeds.dtype) 38 | skip_size = self.prompt_size 39 | else: 40 | skip_size = 0 41 | input_embeds = None 42 | # mask = None 43 | 44 | output, cache, value = self.model(input_ids, input_embeds, attention_mask, cache) 45 | output = output[:, skip_size:, :] 46 | value = value[:, skip_size:, :] 47 | return output, cache, value 48 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/pytorch_baseline/README.md: -------------------------------------------------------------------------------- 1 | This sub-directory houses all of the PyTorch versions of the MLX scripts in the main directory of this repo. 2 | They include: 3 | 4 | ### `pytorch_sft.py` 5 | A fine-tuning script. This loads in data from `data_utils.py`, exact same as the `sft.py` script with MLX. 6 | Pre-trained models are loaded in with [transformers](https://huggingface.co/docs/transformers/index). 7 | Fine-tuning is done with the [PEFT library](https://huggingface.co/docs/peft/en/index) and [LoRAs](https://huggingface.co/docs/peft/en/task_guides/lora_based_methods). 8 | 9 | Trained models are saved with the `.save_pretrained()` function into this directory. 10 | 11 | ### `pytorch_ppo_trainer.py` 12 | This is a gutting/rewriting of the [PPO_Trainer](https://huggingface.co/docs/trl/main/en/ppo_trainer) from HuggingFace's [TRL library](https://huggingface.co/docs/trl/index). 13 | It _should_ match the MLX version as closely as possible. 14 | 15 | ### `pytorch_ppo_training.py` 16 | This is the launcher script for running RLHF/RLAIF with PyTorch, using the `pytorch_ppo_trainer`. 17 | 18 | ### `pytorch_talk_to_model.py` 19 | This loads in a pretrained model with PyTorch and runs it in the terminal for you to live-chat with the model. Good for testing out how well things are working. 20 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/pytorch_baseline/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/pytorch_baseline/__init__.py -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/pytorch_baseline/pytorch_talk_to_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created by Andrew Silva on 5/11/2024 3 | """ 4 | import torch 5 | from transformers import AutoModelForCausalLM, AutoTokenizer 6 | import argparse 7 | 8 | if __name__ == "__main__": 9 | arg_parse = argparse.ArgumentParser(description="Talk to a trained model.") 10 | arg_parse.add_argument( 11 | "--model", 12 | default="./sft_fine_tune/", 13 | help="The path to the local model directory or Hugging Face repo.", 14 | ) 15 | arg_parse.add_argument( 16 | "--tokenizer", 17 | default="meta-llama/Llama-2-7b-chat-hf", 18 | # "meta-llama/Llama-2-7b-chat-hf", "TinyLlama/TinyLlama-1.1B-Chat-v1.0" 19 | help="The path to the local model directory or Hugging Face repo.", 20 | ) 21 | # Generation args 22 | arg_parse.add_argument( 23 | "--max-tokens", 24 | "-m", 25 | type=int, 26 | default=32, 27 | help="The maximum number of tokens to generate", 28 | ) 29 | arg_parse.add_argument( 30 | "--max-context", 31 | '-c', 32 | type=int, 33 | default=1024, 34 | help="The maximum number of tokens from the ongoing conversation that should be wrapped up as context" 35 | ) 36 | arg_parse.add_argument( 37 | "--temp", type=float, default=0.0, help="The sampling temperature" 38 | ) 39 | args = arg_parse.parse_args() 40 | 41 | DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') 42 | tokenizer = AutoTokenizer.from_pretrained(args.tokenizer) 43 | model = AutoModelForCausalLM.from_pretrained(args.model).to(DEVICE) 44 | 45 | print("Type your message to the chat bot below:") 46 | output_message = '' 47 | while True: 48 | input_str = input(">>>") 49 | input_message = f'{output_message}\nUser: {input_str}\nSystem:' 50 | input_message = tokenizer(input_message) 51 | input_message = input_message['input_ids'][-args.max_context:] 52 | with torch.no_grad(): 53 | output_message = model.generate( 54 | input_ids=torch.tensor(input_message, device=DEVICE).unsqueeze(0), 55 | max_new_tokens=args.max_tokens, 56 | temperature=args.temp, 57 | do_sample=args.temp > 0 58 | ) 59 | output_message = tokenizer.decode(output_message[0, len(input_message):]) 60 | output_message = f'System: {output_message.split("User:")[0].split("")[0]}' 61 | print(f'{output_message}') 62 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/requirements.txt: -------------------------------------------------------------------------------- 1 | mlx>=0.14.0 2 | mlx-lm>=0.14.0 3 | transformers>=4.39.3 4 | numpy>=1.26.3 5 | wandb>=0.16.3 6 | datasets>=2.17.0 7 | packaging>=23.2 8 | matplotlib>=3.8.2 -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/mlx-rlhf/talk_to_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created by Andrew Silva on 5/11/2024 3 | """ 4 | import mlx.core as mx 5 | from utils import get_model_and_tokenizer 6 | import argparse 7 | 8 | if __name__ == "__main__": 9 | arg_parse = argparse.ArgumentParser(description="Talk to a trained model.") 10 | arg_parse.add_argument( 11 | "--model", 12 | default="andrewsilva/increasing_digit_fine_tune", 13 | help="The path to the local model directory or Hugging Face repo.", 14 | ) 15 | arg_parse.add_argument( 16 | "--resume-file", 17 | default="digit_fine_tune.npz", 18 | help="Adapter file location" 19 | ) 20 | arg_parse.add_argument( 21 | "--quantize", 22 | action="store_true", 23 | help="Should the model be quantized?" 24 | ) 25 | # Generation args 26 | arg_parse.add_argument( 27 | "--max-tokens", 28 | "-m", 29 | type=int, 30 | default=32, 31 | help="The maximum number of tokens to generate", 32 | ) 33 | arg_parse.add_argument( 34 | "--max-context", 35 | '-c', 36 | type=int, 37 | default=1024, 38 | help="The maximum number of tokens from the ongoing conversation that should be wrapped up as context" 39 | ) 40 | arg_parse.add_argument( 41 | "--temp", type=float, default=0.0, help="The sampling temperature" 42 | ) 43 | args = arg_parse.parse_args() 44 | model, tokenizer = get_model_and_tokenizer(args, need_generate=True, add_peft=False) 45 | 46 | print("Type your message to the chat bot below:") 47 | output_message = '' 48 | while True: 49 | input_str = input(">>>") 50 | input_message = f'{output_message}\nUser: {input_str}\nSystem:' 51 | # input_message = input_str 52 | input_message = tokenizer(input_message) 53 | input_message = mx.array(input_message['input_ids'][-args.max_context:])[None] 54 | output_message = [] 55 | for token in model.generate(input_message, args.temp): 56 | output_message.append(token.item()) 57 | if len(output_message) >= args.max_tokens: 58 | break 59 | output_message = tokenizer.decode(output_message[len(input_message):]) 60 | output_message = f'System: {output_message.split("User:")[0].split("")[0]}' 61 | print(f'{output_message}') 62 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_rlaif_trainer/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | uv pip install mlx==0.25.2 --upgrade 3 | uv pip install "mlx-lm==0.24.0" --upgrade 4 | uv pip install "mlx_embedding_models==0.0.11" 5 | uv pip install --upgrade wandb 6 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_server/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Apple MLX Server", 3 | "uniqueId": "mlx_server", 4 | "description": "MLX Machine learning research on your laptop or in a data center - by Apple", 5 | "plugin-format": "python", 6 | "type": "loader", 7 | "version": "0.1.34", 8 | "supports": [ 9 | "chat", 10 | "completion", 11 | "visualize_model", 12 | "model_layers", 13 | "rag", 14 | "tools", 15 | "template", 16 | "embeddings", 17 | "tokenize", 18 | "logprobs", 19 | "batched" 20 | ], 21 | "model_architectures": [ 22 | "MLX", 23 | "CohereForCausalLM", 24 | "DeepseekV2ForCausalLM", 25 | "GemmaForCausalLM", 26 | "Gemma2ForCausalLM", 27 | "Gemma3ForCausalLM", 28 | "Gemma3ForConditionalGeneration", 29 | "GPTBigCodeForCausalLM", 30 | "LlamaForCausalLM", 31 | "MistralForCausalLM", 32 | "MixtralForCausalLM", 33 | "PhiForCausalLM", 34 | "Phi3ForCausalLM", 35 | "Qwen2ForCausalLM", 36 | "Qwen3ForCausalLM", 37 | "Qwen3MoeForCausalLM" 38 | ], 39 | "supported_hardware_architectures": ["mlx"], 40 | "files": ["main.py", "setup.sh"], 41 | "setup-script": "setup.sh", 42 | "parameters": { 43 | "context_length": { 44 | "type": "integer", 45 | "default": 4096, 46 | "title": "Model Context Length (Fallback)" 47 | } 48 | }, 49 | "parameters_ui": { 50 | "context_length": { 51 | "ui:help": "This context length will be used when the context length is not specified in the model config." 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_server/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | uv pip install "mlx==0.25.2" --upgrade 3 | uv pip install "mlx-lm==0.24.0" --upgrade 4 | uv pip install "mlx_embedding_models==0.0.11" 5 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_vlm_server/generate.py: -------------------------------------------------------------------------------- 1 | # generate.py 2 | # Copyright © 2024 Apple Inc. 3 | 4 | import argparse 5 | import codecs 6 | 7 | import mlx.core as mx 8 | from transformers import AutoProcessor 9 | 10 | from llava import LlavaModel 11 | 12 | 13 | def parse_arguments(): 14 | parser = argparse.ArgumentParser(description="Generate text from an image using a model.") 15 | parser.add_argument( 16 | "--model", 17 | type=str, 18 | default="llava-hf/llava-1.5-7b-hf", 19 | help="The path to the local model directory or Hugging Face repo.", 20 | ) 21 | parser.add_argument( 22 | "--image", 23 | type=str, 24 | default=None, 25 | help="URL or path of the image to process. Leave empty to process text only.", 26 | ) 27 | parser.add_argument( 28 | "--prompt", 29 | type=str, 30 | default="USER: \nWhat are these?\nASSISTANT:", 31 | help="Message to be processed by the model.", 32 | ) 33 | parser.add_argument( 34 | "--max-tokens", 35 | type=int, 36 | default=100, 37 | help="Maximum number of tokens to generate.", 38 | ) 39 | parser.add_argument("--temp", type=float, default=0.3, help="Temperature for sampling.") 40 | parser.add_argument( 41 | "--eos-token", 42 | type=str, 43 | default=None, 44 | help="End of sequence token for tokenizer", 45 | ) 46 | return parser.parse_args() 47 | 48 | 49 | def prepare_inputs(processor, image, prompt): 50 | if image: 51 | inputs = processor(prompt, image) 52 | pixel_values = mx.array(inputs["pixel_values"]) 53 | else: 54 | inputs = processor(prompt) 55 | pixel_values = None # No image provided 56 | input_ids = mx.array(inputs["input_ids"]) 57 | return input_ids, pixel_values 58 | 59 | 60 | def load_model(model_path, tokenizer_config={}): 61 | processor = AutoProcessor.from_pretrained(model_path, **tokenizer_config) 62 | model = LlavaModel.from_pretrained(model_path) 63 | return processor, model 64 | 65 | 66 | def sample(logits, temperature=0.0): 67 | if temperature == 0: 68 | return mx.argmax(logits, axis=-1) 69 | else: 70 | return mx.random.categorical(logits * (1 / temperature)) 71 | 72 | 73 | def generate_text(input_ids, pixel_values, model, processor, max_tokens, temperature): 74 | if pixel_values is not None: 75 | logits, cache = model(input_ids, pixel_values) 76 | else: 77 | logits, cache = model.language_model(input_ids) 78 | logits = logits[:, -1, :] 79 | y = sample(logits, temperature=temperature) 80 | token = y.item() 81 | tokens = [token] 82 | yield token 83 | for n in range(max_tokens - 1): 84 | logits, cache = model.language_model(y[None], cache=cache) 85 | logits = logits[:, -1, :] 86 | y = sample(logits, temperature) 87 | token = y.item() 88 | tokens.append(token) 89 | yield token 90 | 91 | 92 | def main(): 93 | args = parse_arguments() 94 | 95 | tokenizer_config = {} 96 | if args.eos_token is not None: 97 | tokenizer_config["eos_token"] = args.eos_token 98 | 99 | processor, model = load_model(args.model, tokenizer_config) 100 | 101 | prompt = codecs.decode(args.prompt, "unicode_escape") 102 | 103 | input_ids, pixel_values = prepare_inputs(processor, args.image, prompt) 104 | 105 | print(prompt) 106 | 107 | 108 | if __name__ == "__main__": 109 | main() 110 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_vlm_server/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MLX Vision Server", 3 | "uniqueId": "mlx_vlm_server", 4 | "description": "MLX Machine learning multimodal research on your laptop or in a data center - by Blaizzy", 5 | "plugin-format": "python", 6 | "type": "loader", 7 | "version": "0.1.19", 8 | "model_architectures": ["LlavaForConditionalGeneration"], 9 | "supported_hardware_architectures": [ 10 | "mlx" 11 | ], 12 | "files": ["main.py", "setup.sh"], 13 | "setup-script": "setup.sh", 14 | "parameters": {} 15 | } 16 | -------------------------------------------------------------------------------- /transformerlab/plugins/mlx_vlm_server/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | uv pip install mlx==0.23.2 --upgrade 3 | uv pip install "mlx-lm==0.22.1" --upgrade 4 | # pip install numpy transformers torch huggingface_hub Pillow -------------------------------------------------------------------------------- /transformerlab/plugins/nanotron_pretrainer/setup.sh: -------------------------------------------------------------------------------- 1 | git clone https://github.com/huggingface/nanotron 2 | cd nanotron 3 | uv pip install -e . 4 | 5 | uv pip install numba datatrove 6 | uv pip install ninja 7 | uv pip install https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp311-cp311-linux_x86_64.whl 8 | -------------------------------------------------------------------------------- /transformerlab/plugins/ollama_server/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Ollama Server", 3 | "uniqueId": "ollama_server", 4 | "description": "Connects to your instance of ollama to run GGUF models that can host models across GPU and/or CPU.", 5 | "plugin-format": "python", 6 | "type": "loader", 7 | "version": "0.1.3", 8 | "model_architectures": ["GGUF"], 9 | "files": ["main.py", "setup.sh"], 10 | "setup-script": "setup.sh", 11 | "parameters": {}, 12 | "supported_hardware_architectures": ["cpu", "cuda", "mlx", "amd"] 13 | } 14 | -------------------------------------------------------------------------------- /transformerlab/plugins/ollama_server/setup.sh: -------------------------------------------------------------------------------- 1 | # You need ollama installed locally for this to work 2 | # To install ollama do: 3 | # curl https://ollama.ai/install.sh | sh 4 | 5 | uv pip install ollama -------------------------------------------------------------------------------- /transformerlab/plugins/red_teaming/info.md: -------------------------------------------------------------------------------- 1 | # Red Teaming LLMs 2 | 3 | ## Overview 4 | 5 | This plugin helps you evaluate Language Models (LLMs) for vulnerabilities and weaknesses through red teaming techniques. It systematically tests for various security concerns including bias, misinformation, PII leakage, and unauthorized access attempts. 6 | 7 | ## Features 8 | 9 | ### Comprehensive Vulnerability Testing 10 | 11 | Test your LLMs across multiple security domains: 12 | 13 | - **Bias Detection**: Gender, race, religion, and political bias 14 | - **Misinformation**: Factual errors, unsupported claims, expertise misrepresentation 15 | - **PII Leakage**: Database access, direct leakage, session leaks, social manipulation 16 | - **Personal Safety**: Self-harm, bullying, unsafe practices, dangerous challenges 17 | - **Toxicity**: Profanity, insults, threats, mockery 18 | - **Robustness**: Prompt hijacking, input overreliance 19 | - **Unauthorized Access**: 20 | - SQL Injection 21 | - Shell Injection 22 | - Debug Access (Unauthorized debugging capabilities) 23 | - SSRF (Server-Side Request Forgery) 24 | - RBAC (Role-Based Access Control) bypasses 25 | - BOLA (Broken Object Level Authorization) 26 | - BFLA (Broken Function Level Authorization) 27 | - **Illegal Activity**: Detection of content related to weapons, drugs, cybercrime 28 | - **Graphic Content**: Sexual, graphic, or pornographic content 29 | - **Intellectual Property**: Copyright violations, trademark infringement, patent disclosure 30 | 31 | ### Advanced Attack Techniques 32 | 33 | Employ sophisticated attack methods: 34 | 35 | - **Encoding Techniques**: BASE64, ROT13, LEETSPEAK 36 | - **Jailbreak Patterns**: Crescendo, linear, and tree approaches 37 | - **Advanced Methods**: Gray box attacks, prompt injection, multilingual attacks 38 | - **Specialized Probing**: Math problems, prompt probing 39 | 40 | ## Getting Started 41 | 42 | ### 1. Configure Your Target Model 43 | 44 | - Specify the API endpoint for the model you want to test 45 | - Define the target model's purpose and system prompt to improve testing accuracy 46 | - Set API authentication if required 47 | 48 | ### 2. Select Vulnerability Testing Areas 49 | 50 | Choose which vulnerability categories to test from the comprehensive list, such as: 51 | 52 | - Bias categories (gender, race, etc.) 53 | - Security concerns (unauthorized access attempts) 54 | 55 | ### 3. Choose Attack Enhancement Methods 56 | 57 | Select enhancement techniques to strengthen your red team testing: 58 | 59 | - Encoding methods 60 | - Jailbreak approaches 61 | - Sophisticated attack patterns 62 | 63 | ### 4. Set Test Parameters 64 | 65 | - **Judge Model**: Select which LLM to use for evaluating results 66 | - **Number of Attacks**: Define how many attacks per vulnerability to test 67 | - **Target Details**: Specify the purpose and system prompt of the target model 68 | -------------------------------------------------------------------------------- /transformerlab/plugins/red_teaming/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install deepeval==2.6.4 deepteam==0.0.7 tensorboardX instructor anthropic openai instructor -------------------------------------------------------------------------------- /transformerlab/plugins/reward_modeling_llama_factory/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Reward Modeling - Llama Factory", 3 | "uniqueId": "reward_modeling_llama_factory", 4 | "description": "An implementation of RLHF (Reward Modeling) using Llama Factory.", 5 | "plugin-format": "python", 6 | "type": "trainer", 7 | "version": "0.0.9", 8 | "model_architectures": [ 9 | "GemmaForCausalLM", 10 | "LlamaForCausalLM", 11 | "MistralForCausalLM", 12 | "MixtralForCausalLM", 13 | "PhiForCausalLM", 14 | "Phi3ForCausalLM", 15 | "Qwen2ForCausalLM", 16 | "Qwen3ForCausalLM", 17 | "Qwen3MoeForCausalLM" 18 | ], 19 | "supported_hardware_architectures": ["cuda", "amd"], 20 | "files": ["main.py", "setup.sh"], 21 | "setup-script": "setup.sh", 22 | "training_template_format": "none", 23 | "training_data_instructions": "Enxure that the dataset below has the following required columns: \"conversations\", \"chosen\", \"rejected\"", 24 | "parameters": { 25 | "learning_rate": { 26 | "title": "Learning Rate", 27 | "type": "number", 28 | "default": 0.001, 29 | "minimum": 1e-6 30 | }, 31 | "num_train_epochs": { 32 | "title": "Number of Training Epochs", 33 | "type": "integer", 34 | "default": 1, 35 | "minimum": 1 36 | }, 37 | "max_steps": { 38 | "title": "Max Steps (-1 means no limit)", 39 | "type": "integer", 40 | "default": -1 41 | }, 42 | "adaptor_name": { 43 | "title": "Adaptor Name", 44 | "type": "string", 45 | "required": true, 46 | "default": "adaptor" 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /transformerlab/plugins/reward_modeling_llama_factory/info.md: -------------------------------------------------------------------------------- 1 | # RLHF Reward Modeling 2 | ## powered by Llama Factory 3 | 4 | ### Introduction 5 | 6 | In this plugin we implement RLHF using reward modeling. The plugin uses Llama Factory (https://github.com/hiyouga/LLaMA-Factory) under the covers. 7 | 8 | ### Data format 9 | 10 | Llama Factory requires data to be structured in a very specific format. It requires a better response in chosen column and a worse response in rejected column. 11 | 21 | ```json 22 | [ 23 | { 24 | "conversations": "conversation before response (required)", 25 | "chosen": "preferred answer (required)", 26 | "rejected": "rejected answer (required)" 27 | } 28 | ] 29 | ``` 30 | 31 | An example dataset is here: 32 | 33 | dpo_en_demo.json 34 | 35 | 36 | 37 | ### What is Reward Modeling 38 | 39 | In machine learning, reinforcement learning from human feedback (RLHF) is a technique to align an intelligent agent to human preferences. In classical reinforcement learning, the goal of such an agent is to learn a function that guides its behavior called a policy. This function learns to maximize the reward it receives from a separate reward function based on its task performance.[1] However, it is difficult to define explicitly a reward function that approximates human preferences. Therefore, RLHF seeks to train a "reward model" directly from human feedback.[2] The reward model is first trained in a supervised fashion—independently from the policy being optimized—to predict if a response to a given prompt is good (high reward) or bad (low reward) based on ranking data collected from human annotators. This model is then used as a reward function to improve an agent's policy through an optimization algorithm like proximal policy optimization. 40 | 41 | (from wikipedia https://en.wikipedia.org/wiki/Reinforcement_learning_from_human_feedback) 42 | 43 | -------------------------------------------------------------------------------- /transformerlab/plugins/reward_modeling_llama_factory/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install "trl>=0.8.2" 2 | rm -rf LLaMA-Factory 3 | git clone https://github.com/hiyouga/LLaMA-Factory.git 4 | cd LLaMA-Factory 5 | git checkout beec77a0898a39d94f41c23920415f5b4873a23a # this is a known good version 6 | uv pip install -e .[torch,metrics] -------------------------------------------------------------------------------- /transformerlab/plugins/sample_plugin/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Sample Plugin, For Testing", 3 | "uniqueId": "sample_plugin", 4 | "description": "This plugin just takes in the inputs provided, and prints them out. Use it as a template to see what parameters are available to a script.", 5 | "plugin-format": "python", 6 | "type": "trainer", 7 | "version": "1.0.1", 8 | "model_architectures": [ 9 | "LlamaForCausalLM" 10 | ], 11 | "supported_hardware_architectures": [ 12 | "cpu", 13 | "cuda", 14 | "mlx" 15 | ], 16 | "files": [ 17 | "main.py", 18 | "setup.sh" 19 | ], 20 | "setup-script": "setup.sh", 21 | "parameters": { 22 | "adaptor_name": { 23 | "title": "Adaptor Name", 24 | "type": "string", 25 | "required": true 26 | }, 27 | "model": { 28 | "title": "Model", 29 | "type": "string", 30 | "default": "EleutherAI/gpt-neo-2.7B", 31 | "required": "true" 32 | }, 33 | "model_type": { 34 | "title": "Model Type", 35 | "type": "string", 36 | "default": "hf-causal", 37 | "required": "true" 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /transformerlab/plugins/sample_plugin/main.py: -------------------------------------------------------------------------------- 1 | # This is a sample plugin that just displays the parameters that are provided to it 2 | # It is used to test the plugin system 3 | import argparse 4 | import json 5 | 6 | # Connect to the LLM Lab database (you can use this to update job status in the jobs table) 7 | # llmlab_root_dir = os.getenv('LLM_LAB_ROOT_PATH') 8 | # db = sqlite3.connect(llmlab_root_dir + "/workspace/llmlab.sqlite3") 9 | 10 | 11 | def main(): 12 | # Get all arguments provided to this script using argparse 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--input_file", type=str) 15 | args, unknown = parser.parse_known_args() 16 | 17 | with open(args.input_file) as json_file: 18 | input = json.load(json_file) 19 | 20 | print("Input to Script:") 21 | print(input) 22 | 23 | 24 | if __name__ == "__main__": 25 | main() 26 | -------------------------------------------------------------------------------- /transformerlab/plugins/sample_plugin/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | -------------------------------------------------------------------------------- /transformerlab/plugins/sft_llama_factory/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SFT Training - Llama Factory", 3 | "uniqueId": "sft_llama_factory", 4 | "description": "An implementation of Supervised Finetuning using Llama Factory.", 5 | "plugin-format": "python", 6 | "type": "trainer", 7 | "version": "0.0.12", 8 | "model_architectures": [ 9 | "GemmaForCausalLM", 10 | "LlamaForCausalLM", 11 | "MistralForCausalLM", 12 | "MixtralForCausalLM", 13 | "PhiForCausalLM", 14 | "Phi3ForCausalLM", 15 | "Qwen2ForCausalLM", 16 | "Qwen3ForCausalLM", 17 | "Qwen3MoeForCausalLM" 18 | ], 19 | "supported_hardware_architectures": ["cuda", "amd"], 20 | "files": ["main.py", "setup.sh"], 21 | "setup-script": "setup.sh", 22 | "training_template_format": "alpaca", 23 | "parameters": { 24 | "maximum_sequence_length": { 25 | "title": "Maximum Sequence Length", 26 | "type": "integer", 27 | "default": 2048, 28 | "minimum": 1 29 | }, 30 | "learning_rate": { 31 | "title": "Learning Rate", 32 | "type": "number", 33 | "default": 0.001, 34 | "minimum": 1e-6 35 | }, 36 | "num_train_epochs": { 37 | "title": "Number of Training Epochs", 38 | "type": "integer", 39 | "default": 1, 40 | "minimum": 1 41 | }, 42 | "max_steps": { 43 | "title": "Max Steps (-1 means no limit)", 44 | "type": "integer", 45 | "default": -1 46 | }, 47 | "max_samples": { 48 | "title": "Max Samples (-1 means no limit)", 49 | "type": "integer", 50 | "default": -1 51 | }, 52 | "lora_r": { 53 | "title": "Lora R", 54 | "type": "number", 55 | "minimum": 4, 56 | "multipleOf": 4, 57 | "default": 16 58 | }, 59 | "lora_alpha": { 60 | "title": "Lora Alpha", 61 | "type": "number", 62 | "minimum": 4, 63 | "multipleOf": 4, 64 | "default": 32 65 | }, 66 | "lora_dropout": { 67 | "title": "Lora Dropout", 68 | "type": "number", 69 | "minimum": 0.05, 70 | "maximum": 0.9, 71 | "default": 0.05 72 | }, 73 | "adaptor_name": { 74 | "title": "Adaptor Name", 75 | "type": "string", 76 | "required": true, 77 | "default": "adaptor" 78 | } 79 | }, 80 | "parameters_ui": { 81 | "maximum_sequence_length": { 82 | "ui:help": "Maximum sequence length for the model. Longer sequences will be truncated. Keep lower to save memory." 83 | }, 84 | "lora_r": { 85 | "ui:help": "Rank of the update matrices, expressed in int. Lower rank results in smaller update matrices with fewer trainable parameters." 86 | }, 87 | "lora_alpha": { 88 | "ui:help": "LoRA scaling factor. Make it a multiple of LoRA R." 89 | }, 90 | "max_samples": { 91 | "ui:help": "Maximum number of samples to train on. Set to -1 for no limit." 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /transformerlab/plugins/sft_llama_factory/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install "trl>=0.8.2" 2 | rm -rf LLaMA-Factory 3 | git clone https://github.com/hiyouga/LLaMA-Factory.git 4 | cd LLaMA-Factory 5 | git checkout beec77a0898a39d94f41c23920415f5b4873a23a # this is a known good version 6 | uv pip install -e .[torch,metrics] -------------------------------------------------------------------------------- /transformerlab/plugins/synthesizer_docs/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install deepeval==2.8.2 langchain-openai==0.3.16 langchain==0.3.25 langchain-community==0.3.23 langchain_text_splitters==0.3.8 chromadb==1.0.8 sentence-transformers==3.4.1 pypdf==5.4.0 instructor==1.7.9 anthropic==0.49.0 datasets==3.5.1 2 | -------------------------------------------------------------------------------- /transformerlab/plugins/synthesizer_raw_text/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Generate Data from Raw Text", 3 | "uniqueId": "synthesizer_raw_text", 4 | "description": "Use LLMs to create synthetic data for your usecases from reference context. Paste all your reference context to generate a dataset.", 5 | "plugin-format": "python", 6 | "type": "generator", 7 | "version": "0.1.12", 8 | "git": "https://github.com/confident-ai/deepeval", 9 | "url": "https://github.com/confident-ai/deepeval", 10 | "files": ["main.py", "setup.sh"], 11 | "supported_hardware_architectures": ["cpu", "cuda", "mlx", "amd"], 12 | "_dataset": false, 13 | "setup-script": "setup.sh", 14 | "parameters": { 15 | "generation_model": { 16 | "title": "Generation Model", 17 | "type": "string" 18 | }, 19 | "num_goldens": { 20 | "title": "Number of Samples to generate", 21 | "type": "integer", 22 | "minimum": 1, 23 | "default": 10, 24 | "multipleOf": 10 25 | }, 26 | "tflabcustomui_context": { 27 | "title": "Reference Text. Paste all your reference context here", 28 | "type": "string" 29 | } 30 | }, 31 | "parameters_ui": { 32 | "num_goldens": { 33 | "ui:help": "Number of samples to generate", 34 | "ui:widget": "RangeWidget" 35 | }, 36 | "generation_model": { 37 | "ui:help": "Select the model to be used for generation from the drop-down list. (Select `local` to use the local model running)", 38 | "ui:widget": "ModelProviderWidget", 39 | "ui:options": { 40 | "multiple": false 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /transformerlab/plugins/synthesizer_raw_text/info.md: -------------------------------------------------------------------------------- 1 | # Data Synthesizer 2 | 3 | ## Overview 4 | 5 | The Data Synthesizer plugin uses Large Language Models (LLMs) to create synthetic data for various use cases. This plugin supports different generation types and models, allowing users to generate data based on raw reference text. 6 | 7 | ## Parameters 8 | 9 | ### Generation Model 10 | 11 | - **Type:** string 12 | - **Description:** Select the model to be used for generation. The available options are: 13 | - **Claude 3.5 Haiku** 14 | - **Claude 3.5 Sonnet** 15 | - **OpenAI GPT 4o** 16 | - **OpenAI GPT 4o Mini** 17 | - **Local** 18 | 19 | ### Number of Samples to Generate 20 | 21 | - **Type:** integer 22 | - **Minimum:** 1 23 | - **Maximum:** 1000 24 | - **Default:** 10 25 | - **Description:** Specify the number of samples to generate. 26 | 27 | ### Context 28 | 29 | - **Type:** string 30 | - **Description:** Paste all your reference context here for 'context' generation only. 31 | 32 | ## Usage 33 | 34 | 1. **Select the Generation Type:** Choose the type of generation that best fits your needs from the `generation_type` parameter. 35 | 2. **Specify the Number of Samples:** Enter the number of samples to generate in the `num_goldens` parameter. 36 | 3. **Provide Context:** If using 'context' generation, paste your reference context in the `context` parameter. 37 | -------------------------------------------------------------------------------- /transformerlab/plugins/synthesizer_raw_text/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install deepeval==2.8.2 langchain-openai==0.3.16 langchain==0.3.25 langchain-community==0.3.23 langchain_text_splitters==0.3.8 chromadb==1.0.8 pypdf==5.4.0 instructor==1.7.9 anthropic==0.49.0 datasets==3.5.1 2 | -------------------------------------------------------------------------------- /transformerlab/plugins/synthesizer_scratch/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Generate Data from Scratch", 3 | "uniqueId": "synthesizer_scratch", 4 | "description": "Use LLMs to create synthetic data for your usecases from scratch using just a concept of a dataset.", 5 | "plugin-format": "python", 6 | "type": "generator", 7 | "version": "0.1.13", 8 | "git": "https://github.com/confident-ai/deepeval", 9 | "url": "https://github.com/confident-ai/deepeval", 10 | "files": ["main.py", "setup.sh"], 11 | "supported_hardware_architectures": ["cpu", "cuda", "mlx", "amd"], 12 | "dataset": false, 13 | "setup-script": "setup.sh", 14 | "parameters": { 15 | "generation_model": { 16 | "title": "Generation Model (Model to be used for Generation. Select `local` to use the local model running)", 17 | "type": "string" 18 | }, 19 | "num_goldens": { 20 | "title": "Number of Samples to generate", 21 | "type": "integer", 22 | "minimum": 1, 23 | "default": 10, 24 | "multipleOf": 10 25 | }, 26 | "generate_expected_output": { 27 | "title": "Generate Expected Output", 28 | "type": "string", 29 | "enum": ["Yes", "No"] 30 | }, 31 | "scenario": { 32 | "title": "Scenario. Describe the scenario for which you want to generate data", 33 | "type": "string" 34 | }, 35 | "task": { 36 | "title": "Task. Describe the task for which you want to generate data", 37 | "type": "string" 38 | }, 39 | "input_format": { 40 | "title": "Input Format. Describe the format of the input data which will be sent to the model", 41 | "type": "string" 42 | }, 43 | "expected_output_format": { 44 | "title": "Expected Output Format. Describe the format of the output data which will be received from the model", 45 | "type": "string" 46 | } 47 | }, 48 | "parameters_ui": { 49 | "num_goldens": { 50 | "ui:help": "Number of samples to generate", 51 | "ui:widget": "RangeWidget" 52 | }, 53 | "scenario": { 54 | "ui:widget": "textarea", 55 | "ui:help": "Describe the scenario for which you want to generate data" 56 | }, 57 | "task": { 58 | "ui:widget": "textarea", 59 | "ui:help": "Describe the task for which you want to generate data" 60 | }, 61 | "input_format": { 62 | "ui:widget": "textarea", 63 | "ui:help": "Describe the format of the input data which will be sent to the model" 64 | }, 65 | "expected_output_format": { 66 | "ui:widget": "textarea", 67 | "ui:help": "Describe the format of the output data which will be received from the model" 68 | }, 69 | "generation_model": { 70 | "ui:help": "Select the model to be used for generation", 71 | "ui:widget": "ModelProviderWidget", 72 | "ui:options": { 73 | "multiple": false 74 | } 75 | }, 76 | "generate_expected_output": { 77 | "ui:help": "Select Yes if you want to generate data for the expected_output as well", 78 | "ui:widget": "AutoCompleteWidget", 79 | "ui:options": { 80 | "multiple": false 81 | } 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /transformerlab/plugins/synthesizer_scratch/info.md: -------------------------------------------------------------------------------- 1 | # Data Synthesizer 2 | 3 | ## Overview 4 | 5 | The Data Synthesizer plugin uses Large Language Models (LLMs) to create synthetic data for various use cases. This plugin supports different generation types and models, allowing users to generate data from scratch based on a scenario. 6 | 7 | ## Parameters 8 | 9 | ### Generation Model 10 | 11 | - **Type:** string 12 | - **Description:** Select the model to be used for generation. The available options are: 13 | - **Claude 3.5 Haiku** 14 | - **Claude 3.5 Sonnet** 15 | - **OpenAI GPT 4o** 16 | - **OpenAI GPT 4o Mini** 17 | - **Local** 18 | 19 | ### Number of Samples to Generate 20 | 21 | - **Type:** integer 22 | - **Minimum:** 1 23 | - **Maximum:** 1000 24 | - **Default:** 10 25 | - **Description:** Specify the number of samples to generate. 26 | 27 | ### Scenario 28 | 29 | - **Type:** string 30 | - **Description:** Describe the scenario for which you want to generate data for 'scratch' generation only. 31 | 32 | ### Task 33 | 34 | - **Type:** string 35 | - **Description:** Describe the task for which you want to generate data for 'scratch' generation only. 36 | 37 | ### Input Format 38 | 39 | - **Type:** string 40 | - **Description:** Describe the format of the input data which will be sent to the model for 'scratch' generation only. 41 | 42 | ### Expected Output Format 43 | 44 | - **Type:** string 45 | - **Description:** Describe the format of the output data which will be received from the model for 'scratch' generation only. 46 | 47 | ## Usage 48 | 49 | 1. **Select the Generation Model:** Choose the model to be used for generation from the `generation_model` parameter. 50 | 2. **Specify the Number of Samples:** Enter the number of samples to generate in the `num_goldens` parameter. 51 | 3. **Provide Context:** If using 'context' generation, paste your reference context in the `context` parameter. 52 | 4. **Describe the Scenario:** If using 'scratch' generation, describe the scenario in the `scenario` parameter. 53 | 5. **Describe the Task:** If using 'scratch' generation, describe the task in the `task` parameter. 54 | 6. **Specify the Input Format:** If using 'scratch' generation, describe the input format in the `input_format` parameter. 55 | 7. **Specify the Expected Output Format:** If using 'scratch' generation, describe the expected output format in the `expected_output_format` parameter. 56 | -------------------------------------------------------------------------------- /transformerlab/plugins/synthesizer_scratch/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install deepeval==2.8.2 langchain-openai==0.3.16 langchain==0.3.25 langchain-community==0.3.23 langchain_text_splitters==0.3.8 chromadb==1.0.8 sentence-transformers==3.4.1 pypdf==5.4.0 instructor==1.7.9 anthropic==0.49.0 datasets==3.5.1 2 | -------------------------------------------------------------------------------- /transformerlab/plugins/synthetic_dataset_kit/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Synthetic Dataset Generator (Meta synthetic-data-kit)", 3 | "uniqueId": "synthetic_dataset_kit", 4 | "description": "Generates QA pairs, chain-of-thought, or summaries using Meta's synthetic-data-kit and vLLM-based or proxy-backed models.", 5 | "plugin-format": "python", 6 | "type": "generator", 7 | "version": "0.1.0", 8 | "git": "", 9 | "url": "", 10 | "files": ["main.py", "setup.sh"], 11 | "supported_hardware_architectures": ["cuda", "mlx"], 12 | "_dataset": false, 13 | "setup-script": "setup.sh", 14 | "parameters": { 15 | "generation_model": { 16 | "title": "Generation Model. Only local models can be used", 17 | "type": "string", 18 | "enum": ["local"], 19 | "default": "local", 20 | "required": true 21 | }, 22 | "task_type": { 23 | "title": "Generation Type", 24 | "type": "string", 25 | "enum": ["qa", "cot", "summary"], 26 | "default": "qa", 27 | "required": true 28 | }, 29 | "tflabcustomui_docs": { 30 | "title": "Reference Documents", 31 | "type": "string" 32 | }, 33 | "num_pairs": { 34 | "title": "Number of Pairs to Generate", 35 | "type": "integer", 36 | "default": 25, 37 | "minimum": 1, 38 | "maximum": 500 39 | }, 40 | "curation_threshold": { 41 | "title": "Curation Threshold", 42 | "type": "number", 43 | "default": 7.0, 44 | "minimum": 1.0, 45 | "maximum": 10.0 46 | }, 47 | "output_format": { 48 | "title": "Output Format", 49 | "type": "string", 50 | "enum": ["jsonl", "alpaca", "chatml"], 51 | "default": "jsonl" 52 | }, 53 | "prompt_template": { 54 | "title": "Custom Prompt Template (optional)", 55 | "type": "string" 56 | }, 57 | "vllm_api_base": { 58 | "title": "vLLM Server API Base", 59 | "type": "string", 60 | "default": "http://localhost:8338/v1", 61 | "description": "API endpoint of the vLLM server used by synthetic-data-kit." 62 | } 63 | }, 64 | "parameters_ui": { 65 | "generation_model": { 66 | "ui:help": "Select the model to be used for generating QA pairs" 67 | }, 68 | "task_type": { 69 | "ui:help": "Choose the type of generation: QA pairs, CoT examples, or summaries." 70 | }, 71 | "num_pairs": { 72 | "ui:help": "Number of examples to generate from each document." 73 | }, 74 | "curation_threshold": { 75 | "ui:help": "Minimum score (1-10) for keeping a generated example." 76 | }, 77 | "output_format": { 78 | "ui:help": "Select the output format compatible with your fine-tuning setup." 79 | }, 80 | "prompt_template": { 81 | "ui:help": "Custom prompt template for overriding default generation behavior." 82 | }, 83 | "vllm_api_base": { 84 | "ui:help": "Address of the running vLLM server. If left at default, the plugin will try to start one if needed.", 85 | "ui:widget": "text" 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /transformerlab/plugins/synthetic_dataset_kit/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install synthetic-data-kit "vllm>=0.3.0" pdfminer.six==20221105 beautifulsoup4==4.12.3 pytube==15.0.0 youtube-transcript-api==0.6.1 python-docx==1.1.0 python-pptx==0.6.21 pymupdf==1.23.9 langchain-openai==0.3.16 langchain==0.3.25 langchain-community==0.3.23 langchain_text_splitters==0.3.8 deepeval==2.8.2 2 | -------------------------------------------------------------------------------- /transformerlab/plugins/synthetic_dataset_rag/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Generate Dataset with QA Pairs for RAG Evaluation", 3 | "uniqueId": "synthetic_dataset_rag", 4 | "description": "Generate synthetic question-answer pairs from documents for RAG system evaluation.", 5 | "plugin-format": "python", 6 | "type": "generator", 7 | "version": "0.1.7", 8 | "git": "", 9 | "url": "", 10 | "files": ["main.py", "setup.sh"], 11 | "supported_hardware_architectures": ["cpu", "cuda", "mlx", "amd"], 12 | "_dataset": false, 13 | "setup-script": "setup.sh", 14 | "parameters": { 15 | "generation_model": { 16 | "title": "Generation Model (Model to be used for Generation. Select `local` to use the local model running)", 17 | "type": "string" 18 | }, 19 | "tflabcustomui_docs": { 20 | "title": "Reference Documents", 21 | "type": "string" 22 | }, 23 | "chunk_size": { 24 | "title": "Chunk Size", 25 | "type": "integer", 26 | "default": 256, 27 | "minimum": 64, 28 | "required": true 29 | }, 30 | "chunk_overlap": { 31 | "title": "Chunk Overlap", 32 | "type": "integer", 33 | "default": 200, 34 | "minimum": 0, 35 | "required": true 36 | }, 37 | "n_generations": { 38 | "title": "Number of QA Pairs", 39 | "type": "integer", 40 | "default": 10, 41 | "minimum": 1, 42 | "required": true 43 | } 44 | }, 45 | "parameters_ui": { 46 | "generation_model": { 47 | "ui:help": "Select the model to be used for generating QA pairs", 48 | "ui:widget": "ModelProviderWidget", 49 | "ui:options": { 50 | "multiple": false 51 | } 52 | }, 53 | "chunk_size": { 54 | "ui:help": "Size of document chunks in characters. Larger chunks provide more context but may lead to longer generation times." 55 | }, 56 | "chunk_overlap": { 57 | "ui:help": "Number of characters to overlap between chunks to maintain context continuity." 58 | }, 59 | "n_generations": { 60 | "ui:help": "Number of question-answer pairs to generate. Limited by the number of available document chunks." 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /transformerlab/plugins/synthetic_dataset_rag/setup.sh: -------------------------------------------------------------------------------- 1 | uv pip install anthropic==0.49.0 openai==1.77.0 datasets==3.5.1 PyMuPDF==1.25.5 deepeval==2.8.2 langchain-openai==0.3.16 langchain==0.3.25 langchain-community==0.3.23 langchain_text_splitters==0.3.8 -------------------------------------------------------------------------------- /transformerlab/plugins/t5_lora_trainer/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "T5 LoRA Model Trainer", 3 | "uniqueId": "t5_lora_trainer", 4 | "description": "A training script adapted from https://www.philschmid.de/fine-tune-flan-t5-peft . Works with T5 models using transformers.Se2SecTrainer.", 5 | "plugin-format": "python", 6 | "type": "trainer", 7 | "version": "1.0.6", 8 | "model_architectures": ["T5ForConditionalGeneration"], 9 | "supported_hardware_architectures": ["cuda", "amd"], 10 | "git": "", 11 | "url": "", 12 | "files": ["main.py", "setup.sh"], 13 | "setup-script": "setup.sh", 14 | "parameters": { 15 | "adaptor_name": { 16 | "title": "Adaptor Name", 17 | "type": "string", 18 | "required": true 19 | }, 20 | "learning_rate": { 21 | "title": "Learning Rate", 22 | "type": "number", 23 | "default": 5e-5, 24 | "minimum": 1e-6 25 | }, 26 | "num_train_epochs": { 27 | "title": "Number of Training Epochs", 28 | "type": "integer", 29 | "default": 1, 30 | "minimum": 1 31 | }, 32 | "max_steps": { 33 | "title": "Max Steps (-1 means no limit)", 34 | "type": "integer", 35 | "default": -1 36 | }, 37 | "maximum_sequence_length": { 38 | "title": "Maximum Sequence Length", 39 | "type": "integer", 40 | "default": 2048, 41 | "minimum": 1 42 | }, 43 | "lora_r": { 44 | "title": "Lora R", 45 | "type": "number", 46 | "minimum": 4, 47 | "maximum": 64, 48 | "multipleOf": 4, 49 | "default": 16 50 | }, 51 | "lora_alpha": { 52 | "title": "Lora Alpha", 53 | "type": "number", 54 | "minimum": 4, 55 | "maximum": 128, 56 | "multipleOf": 4, 57 | "default": 32 58 | }, 59 | "lora_dropout": { 60 | "title": "Lora Dropout", 61 | "type": "number", 62 | "minimum": 0.05, 63 | "maximum": 0.9, 64 | "default": 0.05 65 | }, 66 | "log_to_wandb": { 67 | "title": "Log to Weights and Biases", 68 | "type": "boolean", 69 | "default": false, 70 | "required": true 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /transformerlab/plugins/t5_lora_trainer/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # %pip install "datasets==2.9.0" "accelerate==0.21.0" "evaluate==0.4.0" loralib 3 | # %pip install rouge-score tensorboard py7zr -------------------------------------------------------------------------------- /transformerlab/plugins/unsloth_grpo_trainer/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #pip install "datasets==2.9.0" "accelerate==0.21.0" "evaluate==0.4.0" loralib 3 | uv pip install trl==0.15.2 bitsandbytes==0.45.4 unsloth==2025.3.19 4 | #pip install rouge-score tensorboard py7zr 5 | -------------------------------------------------------------------------------- /transformerlab/plugins/vllm_server/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vLLM Server", 3 | "uniqueId": "vllm_server", 4 | "description": "vLLM is a fast and easy-to-use library for LLM inference and serving.", 5 | "plugin-format": "python", 6 | "type": "loader", 7 | "version": "1.0.11", 8 | "model_architectures": [ 9 | "CohereForCausalLM", 10 | "FalconForCausalLM", 11 | "GemmaForCausalLM", 12 | "GPTBigCodeForCausalLM", 13 | "LlamaForCausalLM", 14 | "MistralForCausalLM", 15 | "MixtralForCausalLM", 16 | "PhiForCausalLM", 17 | "Phi3ForCausalLM", 18 | "Qwen2ForCausalLM" 19 | ], 20 | "supported_hardware_architectures": ["cuda"], 21 | "files": ["main.py", "setup.sh"], 22 | "setup-script": "setup.sh", 23 | "parameters": { 24 | "max-model-len": { 25 | "title": "Maximum Model Length", 26 | "type": "integer" 27 | }, 28 | "gpu_memory_utilization": { 29 | "title": "GPU Memory Utilization", 30 | "type": "number", 31 | "default": 0.9, 32 | "minimum": 0.0, 33 | "maximum": 1.0 34 | }, 35 | "model_dtype": { 36 | "title": "Select a specific data type for the model", 37 | "type": "string", 38 | "enum": ["auto", "float16", "bfloat16", "float32"] 39 | } 40 | }, 41 | "parameters_ui": { 42 | "model_dtype": { 43 | "ui:help": "Select a specific data type for the model. This might help with older GPUs that do not support bfloat16" 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /transformerlab/plugins/vllm_server/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import subprocess 5 | import sys 6 | 7 | try: 8 | from transformerlab.plugin import get_python_executable 9 | except ImportError: 10 | from transformerlab.plugin_sdk.transformerlab.plugin import get_python_executable 11 | 12 | 13 | # Get all arguments provided to this script using argparse 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument("--model-path", type=str) 16 | parser.add_argument("--parameters", type=str, default="{}") 17 | args, unknown = parser.parse_known_args() 18 | 19 | model = args.model_path 20 | 21 | llmlab_root_dir = os.getenv("LLM_LAB_ROOT_PATH") 22 | 23 | parameters = args.parameters 24 | parameters = json.loads(parameters) 25 | 26 | print("Starting VLLM Server", file=sys.stderr) 27 | 28 | # Quantization is not yet supported but once it is, we need to add the following to index: 29 | # "quantization": { 30 | # "title": "Quantization", 31 | # "type": "string", 32 | # "enum": [ 33 | # "aqlm", 34 | # "awq", 35 | # "deepspeedfp", 36 | # "fp8", 37 | # "fbgemm_fp8", 38 | # "marlin", 39 | # "gptq_marlin_24", 40 | # "gptq_marlin", 41 | # "awq_marlin", 42 | # "gptq", 43 | # "squeezellm", 44 | # "compressed-tensors", 45 | # "bitsandbytes", 46 | # "None" 47 | # ] 48 | # } 49 | 50 | # Now go through the parameters object and remove the key that is equal to "inferenceEngine": 51 | if "inferenceEngine" in parameters: 52 | del parameters["inferenceEngine"] 53 | 54 | if "max-model-len" in parameters: 55 | if parameters["max-model-len"] == "": 56 | del parameters["max-model-len"] 57 | 58 | if "inferenceEngineFriendlyName" in parameters: 59 | del parameters["inferenceEngineFriendlyName"] 60 | 61 | if "num_gpus" in parameters: 62 | del parameters["num_gpus"] 63 | 64 | # The command to run a VLLM server is: 65 | # python -m vllm.entrypoints.openai.api_server --model facebook/opt-125m 66 | # but we can also run it through FastChat's VLLM integration: 67 | # https://github.com/lm-sys/FastChat/blob/main/docs/vllm_integration.md 68 | 69 | # Get plugin directory 70 | real_plugin_dir = os.path.realpath(os.path.dirname(__file__)) 71 | 72 | # Get Python executable (from venv if available) 73 | python_executable = get_python_executable(real_plugin_dir) 74 | 75 | popen_args = [python_executable, "-m", "fastchat.serve.vllm_worker", "--model-path", model] 76 | model_dtype = parameters.get("model_dtype") 77 | # Set model dtype if provided 78 | if model_dtype is not None and model_dtype != "": 79 | popen_args.extend(["--model_dtype", model_dtype]) 80 | 81 | # Add all parameters to the command 82 | for key, value in parameters.items(): 83 | if key == "model_dtype": 84 | continue 85 | popen_args.extend([f"--{key}", str(value)]) 86 | 87 | print(popen_args) 88 | proc = subprocess.Popen(popen_args, stderr=subprocess.PIPE, stdout=None) 89 | 90 | # save worker process id to file 91 | # this will allow transformer lab to kill it later 92 | with open(f"{llmlab_root_dir}/worker.pid", "w") as f: 93 | f.write(str(proc.pid)) 94 | 95 | # read output: 96 | for line in iter(proc.stderr.readline, b""): 97 | print(line, file=sys.stderr) 98 | 99 | print("VLLM Server exited", file=sys.stderr) 100 | sys.exit(1) # 99 is our code for CUDA OOM 101 | -------------------------------------------------------------------------------- /transformerlab/plugins/vllm_server/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | uv pip install vllm==0.5.4 3 | -------------------------------------------------------------------------------- /transformerlab/plugins/vlm_trainer/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | uv pip install --upgrade evaluate bitsandbytes trl peft -------------------------------------------------------------------------------- /transformerlab/plugins/yourbench_data_gen/setup.sh: -------------------------------------------------------------------------------- 1 | # 1. Clone the repo 2 | git clone https://github.com/huggingface/yourbench.git 3 | cd yourbench 4 | git switch -c eb50e1fff84849e19ecb62b45a25ed6afed46a2e 5 | 6 | # Use uv to install the dependencies 7 | # pip install uv # if you do not have uv already 8 | # uv venv 9 | # source .venv/bin/activate 10 | # uv sync 11 | uv sync 12 | uv pip install -e . 13 | uv pip install deepeval langchain-openai instructor anthropic datasets 14 | # Need this to keep the current versions and preserve my sanity 15 | uv pip install torch==2.6.0 --upgrade 16 | uv pip install textstat evaluate -------------------------------------------------------------------------------- /transformerlab/routers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/routers/__init__.py -------------------------------------------------------------------------------- /transformerlab/routers/config.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | import transformerlab.db as db 3 | 4 | 5 | router = APIRouter(prefix="/config", tags=["config"]) 6 | 7 | 8 | @router.get("/get/{key}", summary="") 9 | async def config_get(key: str): 10 | value = await db.config_get(key=key) 11 | return value 12 | 13 | 14 | @router.get("/set", summary="") 15 | async def config_set(k: str, v: str): 16 | await db.config_set(key=k, value=v) 17 | return {"key": k, "value": v} 18 | -------------------------------------------------------------------------------- /transformerlab/routers/prompts.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from typing import Annotated 4 | from fastapi import APIRouter, Body 5 | 6 | from transformerlab.shared import dirs 7 | from transformerlab.shared.shared import slugify 8 | 9 | router = APIRouter(prefix="/prompts", tags=["prompts"]) 10 | 11 | 12 | @router.get("/list") 13 | async def list_prompts(): 14 | """List the prompt templates available in the prompt gallery""" 15 | 16 | remote_gallery_file = os.path.join(dirs.TFL_SOURCE_CODE_DIR, "transformerlab/galleries/prompt-gallery.json") 17 | 18 | with open(remote_gallery_file, "r") as f: 19 | prompt_gallery = json.load(f) 20 | 21 | prompt_templates = [] 22 | prompts_dir = dirs.PROMPT_TEMPLATES_DIR 23 | for file in os.listdir(prompts_dir): 24 | if file.endswith(".json"): 25 | with open(os.path.join(prompts_dir, file), "r") as f: 26 | try: 27 | prompt = json.load(f) 28 | prompt["source"] = "local" 29 | prompt_templates.append(prompt) 30 | except Exception as e: 31 | print(f"Error loading prompt template from file: {file}: {e}, skipping") 32 | 33 | return prompt_gallery + prompt_templates 34 | 35 | 36 | @router.post("/new") 37 | async def new_prompt(title: Annotated[str, Body()], text: Annotated[str, Body()]): 38 | """Create a new prompt template""" 39 | 40 | if "{text}" not in text: 41 | return {"status": "error", "message": "The text must include the placeholder {text}"} 42 | 43 | slug = slugify(title) 44 | prompts_dir = dirs.PROMPT_TEMPLATES_DIR 45 | 46 | prompt_file = os.path.join(prompts_dir, f"{slug}.json") 47 | 48 | json_str = "{}" 49 | 50 | with open(prompt_file, "w") as f: 51 | j = {"id": slug, "title": title, "text": text} 52 | json_str = json.dumps(j, indent=4) 53 | f.write(json_str) 54 | 55 | return {"status": "success", "data": json_str} 56 | 57 | 58 | @router.get("/delete/{prompt_id}") 59 | async def delete_prompt(prompt_id: str): 60 | """Delete a prompt template""" 61 | 62 | prompts_dir = dirs.PROMPT_TEMPLATES_DIR 63 | prompt_file = os.path.join(prompts_dir, f"{prompt_id}.json") 64 | 65 | if os.path.exists(prompt_file): 66 | os.remove(prompt_file) 67 | return {"status": "success", "message": f"Prompt {prompt_id} deleted"} 68 | else: 69 | return {"status": "error", "message": f"Prompt {prompt_id} not found"} 70 | -------------------------------------------------------------------------------- /transformerlab/routers/users.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | 3 | # from .user_service import auth_backend, current_active_user, fastapi_users 4 | 5 | # FastAPI Users integration 6 | from transformerlab.services.user_service import auth_backend, fastapi_users 7 | from transformerlab.schemas.user import UserCreate, UserRead, UserUpdate 8 | 9 | router = APIRouter() 10 | 11 | router.include_router( 12 | fastapi_users.get_auth_router(auth_backend), 13 | prefix="/auth/jwt", 14 | tags=["auth"], 15 | ) 16 | router.include_router( 17 | fastapi_users.get_register_router(UserRead, UserCreate), 18 | prefix="/auth", 19 | tags=["auth"], 20 | ) 21 | router.include_router( 22 | fastapi_users.get_reset_password_router(), 23 | prefix="/auth", 24 | tags=["auth"], 25 | ) 26 | router.include_router( 27 | fastapi_users.get_verify_router(UserRead), 28 | prefix="/auth", 29 | tags=["auth"], 30 | ) 31 | router.include_router( 32 | fastapi_users.get_users_router(UserRead, UserUpdate), 33 | prefix="/users", 34 | tags=["users"], 35 | ) 36 | -------------------------------------------------------------------------------- /transformerlab/schemas/user.py: -------------------------------------------------------------------------------- 1 | """ 2 | User schemas for FastAPI Users integration. 3 | 4 | Any fields we've added to the default FastAPI Users schema need to be added here. 5 | """ 6 | 7 | import uuid 8 | from typing import Optional 9 | 10 | from fastapi_users import schemas 11 | 12 | 13 | class UserRead(schemas.BaseUser[uuid.UUID]): 14 | name: str 15 | 16 | 17 | class UserCreate(schemas.BaseUserCreate): 18 | name: Optional[str] = "" 19 | 20 | 21 | class UserUpdate(schemas.BaseUserUpdate): 22 | name: str 23 | -------------------------------------------------------------------------------- /transformerlab/services/user_service.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from typing import Optional 3 | 4 | from fastapi import Depends, Request 5 | from fastapi_users import BaseUserManager, FastAPIUsers, UUIDIDMixin, models 6 | from fastapi_users.authentication import ( 7 | AuthenticationBackend, 8 | BearerTransport, 9 | JWTStrategy, 10 | ) 11 | 12 | from fastapi_users.db import SQLAlchemyUserDatabase 13 | 14 | from transformerlab.db import get_user_db 15 | from transformerlab.shared.models.models import User 16 | 17 | SECRET = "TEMPSECRET" 18 | 19 | 20 | class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]): 21 | reset_password_token_secret = SECRET 22 | verification_token_secret = SECRET 23 | 24 | async def on_after_register(self, user: User, request: Optional[Request] = None): 25 | print(f"User {user.id} has registered.") 26 | 27 | async def on_after_forgot_password(self, user: User, token: str, request: Optional[Request] = None): 28 | print(f"User {user.id} has forgot their password. Reset token: {token}") 29 | 30 | async def on_after_request_verify(self, user: User, token: str, request: Optional[Request] = None): 31 | print(f"Verification requested for user {user.id}. Verification token: {token}") 32 | 33 | 34 | async def get_user_manager(user_db: SQLAlchemyUserDatabase = Depends(get_user_db)): 35 | yield UserManager(user_db) 36 | 37 | 38 | ### TEMP JWT AUTH TO GET THINGS WORKING 39 | 40 | 41 | def get_jwt_strategy() -> JWTStrategy[models.UP, models.ID]: 42 | return JWTStrategy(secret=SECRET, lifetime_seconds=3600) 43 | 44 | 45 | bearer_transport = BearerTransport(tokenUrl="auth/jwt/login") 46 | auth_backend = AuthenticationBackend( 47 | name="jwt", 48 | transport=bearer_transport, 49 | get_strategy=get_jwt_strategy, 50 | ) 51 | 52 | fastapi_users = FastAPIUsers[User, uuid.UUID](get_user_manager, [auth_backend]) 53 | 54 | current_active_user = fastapi_users.current_user(active=True) 55 | -------------------------------------------------------------------------------- /transformerlab/shared/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/shared/__init__.py -------------------------------------------------------------------------------- /transformerlab/shared/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transformerlab/transformerlab-api/2b8bd222ce60792c8f7f834ada1ddc36ad11cb70/transformerlab/shared/models/__init__.py -------------------------------------------------------------------------------- /transformerlab/tools/calculator/main.py: -------------------------------------------------------------------------------- 1 | """ 2 | Basic calculator functions 3 | 4 | Sample math functions for demo purposes. 5 | """ 6 | 7 | 8 | def add(x: float, y: float) -> float: 9 | """ 10 | Get the sum of two numbers. 11 | 12 | Args: 13 | x: The first number of type float to sum. 14 | y: The second number of type float to sum. 15 | Returns: 16 | The sum of x and y as a number of type float. 17 | """ 18 | return float(x) + float(y) 19 | 20 | 21 | def subtract(x: float, y: float) -> float: 22 | """ 23 | Get the difference between two numbers. 24 | 25 | Args: 26 | x: The minuend being subtracted from as a floating point number. 27 | y: The subtrahend being subtracted as a floating point number. 28 | Returns: 29 | The difference of x minus y as a floating point number. 30 | """ 31 | return float(x) - float(y) 32 | 33 | 34 | def multiply(x: float, y: float) -> float: 35 | """ 36 | Get the product you get from multiplying two numbers together. 37 | 38 | Args: 39 | x: The first number of type float to multiply. 40 | y: The second number of type float to multiply. 41 | Returns: 42 | The product of x times y as a number of type float. 43 | """ 44 | return float(x) * float(y) 45 | 46 | 47 | def divide(x: float, y: float) -> float: 48 | """ 49 | Get the quotient from dividing x by y. 50 | 51 | Args: 52 | x: The dividend that is being divided. 53 | y: The divisor which is what the dividend is being divided by. 54 | Returns: 55 | The quotient of x divided by y as a floating point number. 56 | If y is 0 then this returns the string "undefined" 57 | """ 58 | if y == 0: 59 | return "undefined" 60 | return float(x) / float(y) 61 | --------------------------------------------------------------------------------