├── .gitattributes ├── .github └── workflows │ ├── ci.yml │ ├── cmd.yml │ ├── models-pack.yml │ ├── models-quantize.yml │ ├── prune.yml │ └── sync.yml ├── .gitignore ├── .gitmodules ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── build.go ├── buildkit └── frontend │ ├── LICENSE │ ├── doc.go │ ├── ggufpackerfile │ ├── builder │ │ ├── build.go │ │ └── resolvecache.go │ ├── command │ │ └── command.go │ ├── ggufpackerfile2llb │ │ ├── convert.go │ │ ├── image.go │ │ ├── outline.go │ │ └── platform.go │ ├── instructions │ │ ├── bflag.go │ │ ├── commands.go │ │ ├── parse.go │ │ └── support.go │ ├── linter │ │ ├── linter.go │ │ └── ruleset.go │ └── parser │ │ ├── directives.go │ │ ├── errors.go │ │ ├── line_parsers.go │ │ ├── parser.go │ │ └── split_command.go │ ├── ggufpackerui │ ├── attr.go │ ├── build.go │ ├── config.go │ ├── context.go │ ├── namedcontext.go │ └── requests.go │ └── specs │ └── v1 │ └── image.go ├── cmd └── gguf-packer │ ├── README.md │ ├── estimate.go │ ├── go.mod │ ├── go.sum │ ├── inspect.go │ ├── list.go │ ├── llb_dump.go │ ├── llb_frontend.go │ ├── main.go │ ├── pull.go │ ├── remove.go │ └── run.go ├── docs └── assets │ └── dockerhub-ollama-model-cache.jpg ├── examples ├── ggufpackerfiles │ ├── add-from-git │ │ ├── GGUFPackerfile │ │ └── README.md │ ├── add-from-http │ │ ├── GGUFPackerfile │ │ └── README.md │ ├── convert-lora │ │ ├── GGUFPackerfile │ │ └── README.md │ ├── from-alpine │ │ ├── .ggufpackerignore │ │ ├── GGUFPackerfile │ │ ├── README.md │ │ ├── ignore.txt │ │ └── system-prompt.txt │ ├── from-model │ │ ├── GGUFPackerfile │ │ └── README.md │ ├── multi-stages │ │ ├── GGUFPackerfile │ │ └── README.md │ └── single-stage │ │ ├── GGUFPackerfile │ │ └── README.md └── quickstart │ ├── Dockerfile │ └── Dockerfile.infer ├── go.mod ├── go.sum ├── models └── pack │ ├── Dockerfile.embedding │ ├── Dockerfile.embedding-quantize │ ├── Dockerfile.image-to-text │ ├── Dockerfile.image-to-text-quantize │ ├── Dockerfile.text-to-text │ ├── Dockerfile.text-to-text-quantize │ └── matrix.yaml └── util ├── anyx └── any.go ├── funcx └── error.go ├── mapx └── map.go ├── osx ├── browser.go ├── env.go ├── file.go ├── file_mmap.go ├── file_mmap_js.go ├── file_mmap_unix.go ├── file_mmap_windows.go ├── file_mmap_windows_386.go ├── file_mmap_windows_non386.go └── homedir.go ├── ptr └── pointer.go ├── signalx ├── handler.go ├── handler_unix.go └── handler_windows.go └── strconvx └── quote.go /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | 3 | **/go.sum linguist-generated=true 4 | **/zz_generated.*.go linguist-generated=true 5 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | permissions: 4 | contents: read 5 | pull-requests: read 6 | actions: read 7 | 8 | defaults: 9 | run: 10 | shell: bash 11 | 12 | on: 13 | push: 14 | branches: 15 | - "main" 16 | paths-ignore: 17 | - "docs/**" 18 | - "**.md" 19 | - "**.mdx" 20 | - "**.png" 21 | - "**.jpg" 22 | - "examples/**" 23 | - "models/**" 24 | - "!.github/workflows/ci.yml" 25 | pull_request: 26 | branches: 27 | - "main" 28 | paths-ignore: 29 | - "docs/**" 30 | - "**.md" 31 | - "**.mdx" 32 | - "**.png" 33 | - "**.jpg" 34 | - "examples/**" 35 | - "models/**" 36 | - "!.github/workflows/ci.yml" 37 | 38 | jobs: 39 | ci: 40 | timeout-minutes: 15 41 | runs-on: ubuntu-22.04 42 | steps: 43 | - name: Checkout 44 | uses: actions/checkout@v4 45 | with: 46 | fetch-depth: 1 47 | persist-credentials: false 48 | - name: Setup Go 49 | timeout-minutes: 15 50 | uses: actions/setup-go@v5 51 | with: 52 | go-version: "1.22.7" 53 | cache-dependency-path: | 54 | **/go.sum 55 | - name: Setup Toolbox 56 | timeout-minutes: 5 57 | uses: actions/cache@v4 58 | with: 59 | key: toolbox-${{ runner.os }} 60 | path: | 61 | ${{ github.workspace }}/.sbin 62 | - name: Make 63 | run: make ci 64 | env: 65 | LINT_DIRTY: "true" 66 | -------------------------------------------------------------------------------- /.github/workflows/cmd.yml: -------------------------------------------------------------------------------- 1 | name: cmd 2 | 3 | permissions: 4 | contents: write 5 | actions: read 6 | id-token: write 7 | 8 | defaults: 9 | run: 10 | shell: bash 11 | 12 | on: 13 | push: 14 | branches: 15 | - "main" 16 | paths-ignore: 17 | - "docs/**" 18 | - "**.md" 19 | - "**.mdx" 20 | - "**.png" 21 | - "**.jpg" 22 | - "examples/**" 23 | - "models/**" 24 | - "!.github/workflows/ci.yml" 25 | tags: 26 | - "v*.*.*" 27 | 28 | jobs: 29 | build: 30 | timeout-minutes: 15 31 | runs-on: ubuntu-22.04 32 | steps: 33 | - name: Checkout 34 | uses: actions/checkout@v4 35 | with: 36 | fetch-depth: 1 37 | persist-credentials: false 38 | - name: Setup Go 39 | timeout-minutes: 15 40 | uses: actions/setup-go@v5 41 | with: 42 | go-version: "1.22.7" 43 | cache-dependency-path: | 44 | **/go.sum 45 | - name: Make 46 | run: make build 47 | env: 48 | VERSION: "${{ github.ref_name }}" 49 | - name: Upload Artifact 50 | uses: actions/upload-artifact@v4 51 | with: 52 | include-hidden-files: true 53 | path: ${{ github.workspace }}/.dist/* 54 | - name: Release 55 | if: ${{ startsWith(github.ref, 'refs/tags/') }} 56 | uses: softprops/action-gh-release@v2 57 | with: 58 | fail_on_unmatched_files: true 59 | tag_name: "${{ github.ref_name }}" 60 | prerelease: ${{ contains(github.ref, 'rc') }} 61 | files: ${{ github.workspace }}/.dist/* 62 | 63 | publish: 64 | needs: 65 | - build 66 | permissions: 67 | contents: write 68 | actions: read 69 | id-token: write 70 | timeout-minutes: 15 71 | runs-on: ubuntu-22.04 72 | env: 73 | PACKAGE_REGISTRY: "gpustack" 74 | PACKAGE_IMAGE: "gguf-packer" 75 | steps: 76 | - name: Checkout 77 | uses: actions/checkout@v4 78 | with: 79 | fetch-depth: 1 80 | persist-credentials: false 81 | - name: Setup QEMU 82 | uses: docker/setup-qemu-action@v3 83 | with: 84 | image: tonistiigi/binfmt:qemu-v7.0.0 85 | platforms: "arm64" 86 | - name: Setup Buildx 87 | uses: docker/setup-buildx-action@v3 88 | - name: Login DockerHub 89 | uses: docker/login-action@v3 90 | with: 91 | username: ${{ secrets.CI_DOCKERHUB_USERNAME }} 92 | password: ${{ secrets.CI_DOCKERHUB_PASSWORD }} 93 | - name: Download Artifact 94 | uses: actions/download-artifact@v4 95 | with: 96 | path: ${{ github.workspace }}/.dist 97 | merge-multiple: true 98 | - name: Get Metadata 99 | id: metadata 100 | uses: docker/metadata-action@v5 101 | with: 102 | images: "${{ env.PACKAGE_REGISTRY }}/${{ env.PACKAGE_IMAGE }}" 103 | - name: Package 104 | uses: docker/build-push-action@v6 105 | with: 106 | push: true 107 | file: ${{ github.workspace }}/Dockerfile 108 | context: ${{ github.workspace }} 109 | platforms: "linux/amd64,linux/arm64" 110 | tags: ${{ steps.metadata.outputs.tags }} 111 | labels: ${{ steps.metadata.outputs.labels }} 112 | cache-from: | 113 | type=registry,ref=${{ env.PACKAGE_REGISTRY }}/${{ env.PACKAGE_IMAGE }}:build-cache 114 | cache-to: | 115 | type=registry,mode=max,oci-mediatypes=false,compression=gzip,ref=${{ env.PACKAGE_REGISTRY }}/${{ env.PACKAGE_IMAGE }}:build-cache,ignore-error=true 116 | provenance: true 117 | sbom: true 118 | -------------------------------------------------------------------------------- /.github/workflows/models-pack.yml: -------------------------------------------------------------------------------- 1 | name: model-pack 2 | 3 | permissions: 4 | contents: write 5 | actions: read 6 | id-token: write 7 | 8 | defaults: 9 | run: 10 | shell: bash 11 | 12 | on: 13 | workflow_dispatch: 14 | inputs: 15 | model_repository: 16 | description: "Model Repository" 17 | required: true 18 | type: string 19 | default: "" 20 | model_name: 21 | description: "Model Name" 22 | required: true 23 | type: string 24 | default: "" 25 | model_override_tags: 26 | description: "Model Override Tags" 27 | required: false 28 | type: string 29 | default: "" 30 | 31 | jobs: 32 | generate-matrix: 33 | runs-on: ubuntu-22.04 34 | outputs: 35 | matrix: ${{ steps.set-matrix.outputs.matrix }} 36 | steps: 37 | - name: Checkout 38 | uses: actions/checkout@v4 39 | with: 40 | fetch-depth: 1 41 | persist-credentials: false 42 | - name: Set Matrix 43 | id: set-matrix 44 | run: | 45 | #!/usr/bin/env bash 46 | 47 | declare -A OVERRIDE_MODELS_MAP=() 48 | 49 | if [[ -n "${{ github.event.inputs.model_override_tags }}" ]]; then 50 | OVERRIDE_TAGS="$(echo "${{ github.event.inputs.model_override_tags }}" | tr ',' '\n' | jq -R . | jq -s -c .)" 51 | OVERRIDE_MODELS=$(echo "${OVERRIDE_TAGS}" | jq -r '.[] | "docker.io/${{ github.event.inputs.model_repository }}/${{ github.event.inputs.model_name }}:\(.)"') 52 | for MODEL in $OVERRIDE_MODELS; do 53 | OVERRIDE_MODELS_MAP[$MODEL]=$MODEL 54 | done 55 | fi 56 | 57 | CANDIDATES="$(yq 'map(.[] | select(.name == "${{ github.event.inputs.model_name }}"))' --output-format json --indent 0 ${{ github.workspace }}/models/pack/matrix.yaml)" 58 | MODELS=$(echo "${CANDIDATES}" | jq -r '.[] | "docker.io/${{ github.event.inputs.model_repository }}/${{ github.event.inputs.model_name }}:\(.tag)"') 59 | EXISTING_MODELS=() 60 | for MODEL in $MODELS; do 61 | if [[ -v OVERRIDE_MODELS_MAP[$MODEL] ]]; then 62 | continue; 63 | fi 64 | if oras manifest fetch $MODEL &> /dev/null; then 65 | EXISTING_MODELS+=($MODEL) 66 | fi 67 | done 68 | 69 | if [[ ${#EXISTING_MODELS[@]} -eq 0 ]]; then 70 | MATRIX="${CANDIDATES}" 71 | else 72 | MATRIX=$(echo "${CANDIDATES}" | jq -c --argjson existing "$(printf '%s\n' "${EXISTING_MODELS[@]}" | jq -R . | jq -s -c .)" 'map(select("${{ github.event.inputs.model_name }}:\(.tag)" as $img | $existing | index("docker.io/${{ github.event.inputs.model_repository }}/" + $img) | not))') 73 | fi 74 | 75 | echo "matrix=${MATRIX}" >> "$GITHUB_OUTPUT" 76 | 77 | pack: 78 | needs: 79 | - generate-matrix 80 | runs-on: ubuntu-22.04 81 | strategy: 82 | fail-fast: false 83 | matrix: 84 | include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} 85 | steps: 86 | - name: Maximize Docker Build Space 87 | uses: gpustack/.github/.github/actions/maximize-docker-build-space@main 88 | with: 89 | root-reserve-mb: 1024 90 | temp-reserve-mb: 1024 91 | swap-size-mb: 1024 92 | deep-clean: true 93 | - name: Checkout 94 | uses: actions/checkout@v4 95 | with: 96 | fetch-depth: 1 97 | persist-credentials: false 98 | - name: Setup Buildx 99 | uses: docker/setup-buildx-action@v3 100 | - name: Login DockerHub 101 | uses: docker/login-action@v3 102 | with: 103 | username: ${{ secrets.CI_DOCKERHUB_USERNAME }} 104 | password: ${{ secrets.CI_DOCKERHUB_PASSWORD }} 105 | - name: Prepare 106 | id: prepare 107 | run: | 108 | #!/usr/bin/env bash 109 | 110 | echo "Get outputs" 111 | MODEL_VENDOR="$(echo "${{ matrix.repository }}" | cut -d'/' -f1)" 112 | echo "model_vendor=${MODEL_VENDOR}" >> "$GITHUB_OUTPUT" 113 | - name: Package 114 | uses: docker/build-push-action@v6 115 | with: 116 | push: true 117 | context: "${{ github.workspace }}/models/pack" 118 | file: "${{ github.workspace }}/models/pack/Dockerfile.${{ matrix.usage }}" 119 | no-cache: true 120 | build-args: | 121 | MODEL_VENDOR=${{ steps.prepare.outputs.model_vendor }} 122 | MODEL_REPOSITORY=${{ matrix.repository }} 123 | MODEL_FILE=${{ matrix.file }} 124 | ${{ matrix.project_file != '' && format('MODEL_PROJECTOR_FILE={0}', matrix.project_file ) || ''}} 125 | ${{ matrix.quantize_type != '' && format('MODEL_QUANTIZE_TYPE={0}', matrix.quantize_type) || ''}} 126 | ${{ matrix.context_size != '' && format('MODEL_CONTEXT_SIZE={0}', matrix.context_size) || '' }} 127 | tags: | 128 | "${{ github.event.inputs.model_repository }}/${{ matrix.name }}:${{ matrix.tag }}" 129 | - name: Review Space Usage 130 | run: | 131 | #!/usr/bin/env bash 132 | 133 | df -h 134 | -------------------------------------------------------------------------------- /.github/workflows/models-quantize.yml: -------------------------------------------------------------------------------- 1 | name: model-quantize 2 | 3 | permissions: 4 | contents: write 5 | actions: read 6 | id-token: write 7 | 8 | defaults: 9 | run: 10 | shell: bash 11 | 12 | on: 13 | workflow_dispatch: 14 | inputs: 15 | huggingface_repository: 16 | description: "HuggingFace Repository" 17 | required: true 18 | type: string 19 | default: "" 20 | model_convert_type: 21 | description: "Model Convert Type" 22 | required: true 23 | type: string 24 | default: "F16" 25 | model_quantize_types: 26 | description: "Model Quantize Types" 27 | required: true 28 | type: string 29 | default: "Q4_K_M,Q5_K_M" 30 | model_repository: 31 | description: "Model Repository" 32 | required: true 33 | type: string 34 | default: "" 35 | model_name: 36 | description: "Model Name" 37 | required: true 38 | type: string 39 | default: "" 40 | model_tag: 41 | description: "Model Tag" 42 | required: true 43 | type: string 44 | default: "" 45 | model_usage: 46 | description: "Model Usage" 47 | required: true 48 | type: string 49 | default: "text-to-text" 50 | 51 | jobs: 52 | convert: 53 | runs-on: ubuntu-22.04 54 | outputs: 55 | model_suffix: ${{ steps.prepare.outputs.model_suffix }} 56 | matrix: ${{ steps.prepare.outputs.matrix }} 57 | steps: 58 | - name: Maximize Docker Build Space 59 | uses: gpustack/.github/.github/actions/maximize-docker-build-space@main 60 | with: 61 | root-reserve-mb: 1024 62 | temp-reserve-mb: 1024 63 | swap-size-mb: 1024 64 | deep-clean: true 65 | - name: Setup Buildx 66 | uses: docker/setup-buildx-action@v3 67 | with: 68 | driver-opts: | 69 | image=thxcode/buildkit:v0.15.1-git-lfs 70 | - name: Login DockerHub 71 | uses: docker/login-action@v3 72 | with: 73 | username: ${{ secrets.CI_DOCKERHUB_USERNAME }} 74 | password: ${{ secrets.CI_DOCKERHUB_PASSWORD }} 75 | - name: Prepare 76 | id: prepare 77 | env: 78 | HF_TOKEN: ${{ secrets.CI_HUGGINGFACE_TOKEN }} 79 | CONVERT_TYPE: ${{ github.event.inputs.model_convert_type }} 80 | run: | 81 | #!/usr/bin/env bash 82 | 83 | echo "Get Dockerfile" 84 | MODEL_VENDOR="$(echo "${{ github.event.inputs.huggingface_repository }}" | cut -d'/' -f1)" 85 | MODEL_NAME="$(echo "${{ github.event.inputs.huggingface_repository }}" | cut -d'/' -f2)" 86 | cat < ${{ github.workspace }}/Dockerfile 87 | # syntax=gpustack/gguf-packer:latest 88 | FROM scratch AS model 89 | ADD https://gpustack:${HF_TOKEN}@huggingface.co/${{ github.event.inputs.huggingface_repository }}.git ${MODEL_NAME} 90 | FROM scratch 91 | LABEL gguf.model.from="Hugging Face" 92 | LABEL gguf.model.usage="${{ github.event.inputs.model_usage }}" 93 | LABEL gguf.model.vendor="${MODEL_VENDOR}" 94 | CONVERT --from=model --type=${CONVERT_TYPE} ${MODEL_NAME} ${MODEL_NAME}.${CONVERT_TYPE}.gguf 95 | CMD ["-m", "${MODEL_NAME}.${CONVERT_TYPE}.gguf", "-c", "8192", "-np", "4"] 96 | EOF 97 | 98 | echo "Get outputs" 99 | echo "model_suffix=$(echo "${CONVERT_TYPE}" | tr '[:upper:]' '[:lower:]' | sed 's/_/-/g')" >> "$GITHUB_OUTPUT" 100 | echo "matrix=$(echo "${{ github.event.inputs.model_quantize_types }}" | tr ',' '\n' | jq -R . | jq -s -c .)" >> "$GITHUB_OUTPUT" 101 | - name: Package 102 | uses: docker/build-push-action@v6 103 | with: 104 | push: true 105 | context: ${{ github.workspace }} 106 | no-cache: true 107 | tags: | 108 | "${{ github.event.inputs.model_repository }}/${{ github.event.inputs.model_name }}:${{ github.event.inputs.model_tag }}-${{ steps.prepare.outputs.model_suffix }}" 109 | - name: Review Space Usage 110 | run: | 111 | #!/usr/bin/env bash 112 | 113 | df -h 114 | 115 | quantize: 116 | needs: 117 | - convert 118 | runs-on: ubuntu-22.04 119 | strategy: 120 | fail-fast: false 121 | matrix: 122 | type: ${{ fromJson(needs.convert.outputs.matrix) }} 123 | steps: 124 | - name: Maximize Docker Build Space 125 | uses: gpustack/.github/.github/actions/maximize-docker-build-space@main 126 | with: 127 | deep-clean: true 128 | - name: Setup Buildx 129 | uses: docker/setup-buildx-action@v3 130 | - name: Login DockerHub 131 | uses: docker/login-action@v3 132 | with: 133 | username: ${{ secrets.CI_DOCKERHUB_USERNAME }} 134 | password: ${{ secrets.CI_DOCKERHUB_PASSWORD }} 135 | - name: Prepare 136 | id: prepare 137 | env: 138 | CONVERT_TYPE: ${{ github.event.inputs.model_convert_type }} 139 | QUANTIZE_TYPE: ${{ matrix.type }} 140 | run: | 141 | #!/usr/bin/env bash 142 | 143 | echo "Get Dockerfile" 144 | MODEL_VENDOR="$(echo "${{ github.event.inputs.huggingface_repository }}" | cut -d'/' -f1)" 145 | MODEL_NAME="$(echo "${{ github.event.inputs.huggingface_repository }}" | cut -d'/' -f2)" 146 | QUANTIZE_FROM="${{ github.event.inputs.model_repository }}/${{ github.event.inputs.model_name }}:${{ github.event.inputs.model_tag }}-${{ needs.convert.outputs.model_suffix }}" 147 | cat < ${{ github.workspace }}/Dockerfile 148 | # syntax=gpustack/gguf-packer:latest 149 | FROM scratch 150 | LABEL gguf.model.from="Hugging Face" 151 | LABEL gguf.model.usage="${{ github.event.inputs.model_usage }}" 152 | LABEL gguf.model.vendor="${MODEL_VENDOR}" 153 | QUANTIZE --from=${QUANTIZE_FROM} --type=${QUANTIZE_TYPE} ${MODEL_NAME}.${CONVERT_TYPE}.gguf ${MODEL_NAME}.${QUANTIZE_TYPE}.gguf 154 | CMD ["-m", "${MODEL_NAME}.${QUANTIZE_TYPE}.gguf", "-c", "8192", "-np", "4"] 155 | EOF 156 | 157 | echo "Get outputs" 158 | echo "model_suffix=$(echo "${QUANTIZE_TYPE}" | tr '[:upper:]' '[:lower:]' | sed 's/_/-/g')" >> "$GITHUB_OUTPUT" 159 | - name: Package 160 | uses: docker/build-push-action@v6 161 | with: 162 | push: true 163 | context: ${{ github.workspace }} 164 | no-cache: true 165 | tags: | 166 | "${{ github.event.inputs.model_repository }}/${{ github.event.inputs.model_name }}:${{ github.event.inputs.model_tag }}-${{ steps.prepare.outputs.model_suffix }}" 167 | -------------------------------------------------------------------------------- /.github/workflows/prune.yml: -------------------------------------------------------------------------------- 1 | name: prune 2 | 3 | permissions: 4 | contents: write 5 | pull-requests: write 6 | actions: write 7 | issues: write 8 | 9 | defaults: 10 | run: 11 | shell: bash 12 | 13 | on: 14 | workflow_dispatch: 15 | inputs: 16 | prune: 17 | description: 'Prune all caches' 18 | required: false 19 | type: boolean 20 | default: false 21 | schedule: 22 | - cron: "0 0 * * *" # every day at 00:00 UTC 23 | 24 | jobs: 25 | close-stale-issues-and-prs: 26 | uses: gpustack/.github/.github/workflows/close-stale-issues-and-prs.yml@main 27 | 28 | clean-stale-caches: 29 | uses: gpustack/.github/.github/workflows/clean-stale-caches.yml@main 30 | with: 31 | # allow to prune all caches on demand 32 | prune: ${{ github.event_name != 'schedule' && inputs.prune || false }} 33 | -------------------------------------------------------------------------------- /.github/workflows/sync.yml: -------------------------------------------------------------------------------- 1 | name: sync 2 | 3 | permissions: 4 | contents: read 5 | pull-requests: read 6 | actions: read 7 | 8 | defaults: 9 | run: 10 | shell: bash 11 | 12 | on: 13 | workflow_dispatch: 14 | inputs: 15 | max_releases: 16 | description: "Maximum number of latest releases to sync" 17 | required: false 18 | default: 1 19 | type: number 20 | specific_release_tag: 21 | description: "Specific release tag to sync" 22 | required: false 23 | default: "" 24 | type: string 25 | dry_run: 26 | description: "Skip the actual sync" 27 | required: false 28 | default: false 29 | type: boolean 30 | schedule: 31 | - cron: "0 */12 * * *" # every 12 hours 32 | 33 | jobs: 34 | gitcode: 35 | if: github.event_name == 'workflow_dispatch' 36 | runs-on: ubuntu-22.04 37 | timeout-minutes: 240 38 | steps: 39 | - name: Checkout 40 | uses: actions/checkout@v4 41 | with: 42 | fetch-depth: 0 43 | persist-credentials: false 44 | - name: Sync 45 | uses: gpustack/.github/.github/actions/mirror-release-gitcode@main 46 | with: 47 | gitcode-username: "${{ secrets.CI_GITCODE_USERNAME }}" 48 | gitcode-password: "${{ secrets.CI_GITCODE_PASSWORD }}" 49 | gitcode-token: "${{ secrets.CI_GITCODE_TOKEN }}" 50 | max-releases: "${{ inputs.max_releases && inputs.max_releases || '1' }}" 51 | specific-release-tag: "${{ inputs.specific_release_tag && inputs.specific_release_tag || '' }}" 52 | dry-run: "${{ inputs.dry_run && inputs.dry_run || 'false' }}" 53 | 54 | gitee: 55 | if: github.event_name == 'workflow_dispatch' 56 | runs-on: ubuntu-22.04 57 | timeout-minutes: 120 58 | steps: 59 | - name: Checkout 60 | uses: actions/checkout@v4 61 | with: 62 | fetch-depth: 0 63 | persist-credentials: false 64 | - name: Sync 65 | uses: gpustack/.github/.github/actions/mirror-release-gitee@main 66 | with: 67 | gitee-username: "${{ secrets.CI_GITEE_USERNAME }}" 68 | gitee-token: "${{ secrets.CI_GITEE_TOKEN }}" 69 | max-releases: "${{ inputs.max_releases && inputs.max_releases || '1' }}" 70 | specific-release-tag: "${{ inputs.specific_release_tag && inputs.specific_release_tag || '' }}" 71 | dry-run: "${{ inputs.dry_run && inputs.dry_run || 'false' }}" 72 | 73 | tencent-cos: 74 | runs-on: ubuntu-22.04 75 | timeout-minutes: 120 76 | steps: 77 | - name: Sync 78 | uses: gpustack/.github/.github/actions/mirror-release-tencent-cos@main 79 | with: 80 | tencent-secret-id: "${{ secrets.CI_TECENTCOS_SECRET_ID }}" 81 | tencent-secret-key: "${{ secrets.CI_TECENTCOS_SECRET_KEY }}" 82 | tencent-cos-region: "ap-guangzhou" 83 | tencent-cos-bucket: "gpustack-1303613262" 84 | max-releases: "${{ inputs.max_releases && inputs.max_releases || '1' }}" 85 | specific-release-tag: "${{ inputs.specific_release_tag && inputs.specific_release_tag || '' }}" 86 | dry-run: "${{ inputs.dry_run && inputs.dry_run || 'false' }}" 87 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Files 2 | .DS_Store 3 | *.lock 4 | *.test 5 | *.out 6 | *.swp 7 | *.swo 8 | *.db 9 | *.exe 10 | *.exe~ 11 | *.dll 12 | *.so 13 | *.dylib 14 | *.log 15 | go.work 16 | go.work.* 17 | 18 | # Dirs 19 | /.idea 20 | /.vscode 21 | /.kube 22 | /.terraform 23 | /.vagrant 24 | /.bundle 25 | /.cache 26 | /.docker 27 | /.entc 28 | /.sbin 29 | /.dist 30 | /log 31 | /certs 32 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "examples/models/Qwen2-0.5B-Instruct"] 2 | path = examples/models/Qwen2-0.5B-Instruct 3 | url = https://huggingface.co/Qwen/Qwen2-0.5B-Instruct 4 | [submodule "examples/ggufpackerfiles/single-stage-convert-from-local/Qwen2-0.5B-Instruct"] 5 | path = examples/ggufpackerfiles/single-stage-convert-from-local/Qwen2-0.5B-Instruct 6 | url = https://huggingface.co/Qwen/Qwen2-0.5B-Instruct 7 | [submodule "examples/ggufpackerfiles/multi-stages-convert-from-local/Qwen2-0.5B-Instruct"] 8 | path = examples/ggufpackerfiles/multi-stages-convert-from-local/Qwen2-0.5B-Instruct 9 | url = https://huggingface.co/Qwen/Qwen2-0.5B-Instruct 10 | [submodule "examples/ggufpackerfiles/multi-stages/Qwen2-0.5B-Instruct"] 11 | path = examples/ggufpackerfiles/multi-stages/Qwen2-0.5B-Instruct 12 | url = https://huggingface.co/Qwen/Qwen2-0.5B-Instruct 13 | [submodule "examples/ggufpackerfiles/single-stage/Qwen2-0.5B-Instruct"] 14 | path = examples/ggufpackerfiles/single-stage/Qwen2-0.5B-Instruct 15 | url = https://huggingface.co/Qwen/Qwen2-0.5B-Instruct 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1.7-labs 2 | FROM --platform=$TARGETPLATFORM ubuntu:22.04 3 | SHELL ["/bin/bash", "-c"] 4 | 5 | ARG TARGETPLATFORM 6 | ARG TARGETOS 7 | ARG TARGETARCH 8 | 9 | ENV DEBIAN_FRONTEND=noninteractive \ 10 | TZ=UTC \ 11 | LC_ALL=C.UTF-8 12 | RUN </dev/null | tr '[:upper:]' '[:lower:]' || echo "unknown") 10 | 11 | DEPS_UPDATE ?= false 12 | deps: 13 | @echo "+++ $@ +++" 14 | 15 | cd $(SRCDIR) && go mod tidy && go mod download 16 | cd $(SRCDIR)/cmd/gguf-packer && go mod tidy && go mod download 17 | 18 | if [[ "$(DEPS_UPDATE)" == "true" ]]; then \ 19 | cd $(SRCDIR) && go get -u -v ./...; \ 20 | cd $(SRCDIR)/cmd/gguf-packer && go get -u -v ./...; \ 21 | fi 22 | 23 | @echo "--- $@ ---" 24 | 25 | generate: 26 | @echo "+++ $@ +++" 27 | 28 | cd $(SRCDIR) && go generate ./... 29 | cd $(SRCDIR)/cmd/gguf-packer && go generate ./... 30 | 31 | @echo "--- $@ ---" 32 | 33 | LINT_DIRTY ?= false 34 | lint: 35 | @echo "+++ $@ +++" 36 | 37 | if [[ "$(LINT_DIRTY)" == "true" ]]; then \ 38 | if [[ -n $$(git status --porcelain) ]]; then \ 39 | echo "Code tree is dirty."; \ 40 | exit 1; \ 41 | fi; \ 42 | fi 43 | 44 | [[ -d "$(SRCDIR)/.sbin" ]] || mkdir -p "$(SRCDIR)/.sbin" 45 | 46 | [[ -f "$(SRCDIR)/.sbin/goimports-reviser" ]] || \ 47 | curl --retry 3 --retry-all-errors --retry-delay 3 -sSfL "https://github.com/incu6us/goimports-reviser/releases/download/v3.6.5/goimports-reviser_3.6.5_$(GOOS)_$(GOARCH).tar.gz" \ 48 | | tar -zxvf - --directory "$(SRCDIR)/.sbin" --no-same-owner --exclude ./LICENSE --exclude ./README.md && chmod +x "$(SRCDIR)/.sbin/goimports-reviser" 49 | cd $(SRCDIR) && \ 50 | go list -f "{{.Dir}}" ./... | xargs -I {} find {} -maxdepth 1 -type f -name '*.go' ! -name 'gen.*' ! -name 'zz_generated.*' \ 51 | | xargs -I {} "$(SRCDIR)/.sbin/goimports-reviser" -use-cache -imports-order=std,general,company,project,blanked,dotted -output=file {} 52 | cd $(SRCDIR)/cmd/gguf-packer && \ 53 | go list -f "{{.Dir}}" ./... | xargs -I {} find {} -maxdepth 1 -type f -name '*.go' ! -name 'gen.*' ! -name 'zz_generated.*' \ 54 | | xargs -I {} "$(SRCDIR)/.sbin/goimports-reviser" -use-cache -imports-order=std,general,company,project,blanked,dotted -output=file {} 55 | 56 | [[ -f "$(SRCDIR)/.sbin/golangci-lint" ]] || \ 57 | curl --retry 3 --retry-all-errors --retry-delay 3 -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ 58 | | sh -s -- -b "$(SRCDIR)/.sbin" "v1.59.0" 59 | cd $(SRCDIR) && \ 60 | "$(SRCDIR)/.sbin/golangci-lint" run --fix ./... 61 | cd $(SRCDIR)/cmd/gguf-packer && \ 62 | "$(SRCDIR)/.sbin/golangci-lint" run --fix ./... 63 | 64 | @echo "--- $@ ---" 65 | 66 | test: 67 | @echo "+++ $@ +++" 68 | 69 | go test -v -failfast -race -cover -timeout=30m $(SRCDIR)/... 70 | 71 | @echo "--- $@ ---" 72 | 73 | 74 | benchmark: 75 | @echo "+++ $@ +++" 76 | 77 | go test -v -failfast -run="^Benchmark[A-Z]+" -bench=. -benchmem -timeout=30m $(SRCDIR)/... 78 | 79 | @echo "--- $@ ---" 80 | 81 | gguf-packer: 82 | [[ -d "$(SRCDIR)/.dist" ]] || mkdir -p "$(SRCDIR)/.dist" 83 | 84 | cd "$(SRCDIR)/cmd/gguf-packer" && for os in darwin linux windows; do \ 85 | if [[ $$os == "windows" ]]; then \ 86 | suffix=".exe"; \ 87 | else \ 88 | suffix=""; \ 89 | fi; \ 90 | for arch in amd64 arm64; do \ 91 | echo "Building gguf-packer for $$os-$$arch $(VERSION)"; \ 92 | GOOS="$$os" GOARCH="$$arch" CGO_ENABLED=0 go build \ 93 | -trimpath \ 94 | -ldflags="-w -s -X main.Version=$(VERSION)" \ 95 | -tags="urfave_cli_no_docs netgo" \ 96 | -o $(SRCDIR)/.dist/gguf-packer-$$os-$$arch$$suffix; \ 97 | done; \ 98 | if [[ $$os == "darwin" ]]; then \ 99 | [[ -d "$(SRCDIR)/.sbin" ]] || mkdir -p "$(SRCDIR)/.sbin"; \ 100 | [[ -f "$(SRCDIR)/.sbin/lipo" ]] || \ 101 | GOBIN="$(SRCDIR)/.sbin" go install github.com/konoui/lipo@v0.9.1; \ 102 | "$(SRCDIR)/.sbin/lipo" -create -output $(SRCDIR)/.dist/gguf-packer-darwin-universal $(SRCDIR)/.dist/gguf-packer-darwin-amd64 $(SRCDIR)/.dist/gguf-packer-darwin-arm64; \ 103 | fi;\ 104 | if [[ $$os == "$(GOOS)" ]] && [[ $$arch == "$(GOARCH)" ]]; then \ 105 | cp -rf $(SRCDIR)/.dist/gguf-packer-$$os-$$arch$$suffix $(SRCDIR)/.dist/gguf-packer$$suffix; \ 106 | fi; \ 107 | done 108 | 109 | build: gguf-packer 110 | 111 | PACKAGE_PUBLISH ?= false 112 | PACKAGE_REGISTRY ?= "gpustack" 113 | PACKAGE_IMAGE ?= "gguf-packer" 114 | package: build 115 | @echo "+++ $@ +++" 116 | 117 | if [[ -z $$(command -v docker) ]]; then \ 118 | echo "Docker is not installed."; \ 119 | exit 1; \ 120 | fi; \ 121 | platform="linux/amd64,linux/arm64"; \ 122 | image="$(PACKAGE_IMAGE):$(VERSION)"; \ 123 | if [[ -n "$(PACKAGE_REGISTRY)" ]]; then \ 124 | image="$(PACKAGE_REGISTRY)/$$image"; \ 125 | fi; \ 126 | if [[ "$(PACKAGE_PUBLISH)" == "true" ]]; then \ 127 | if [[ -z $$(docker buildx inspect --builder "gguf-packer") ]]; then \ 128 | docker run --rm --privileged tonistiigi/binfmt:qemu-v7.0.0 --install $$platform; \ 129 | docker buildx create --name "gguf-packer" --driver "docker-container" --buildkitd-flags "--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host" --bootstrap; \ 130 | fi; \ 131 | docker buildx build --progress=plain --platform=$$platform --sbom=true --provenance=true --builder="gguf-packer" --output="type=image,name=$$image,push=true" "$(SRCDIR)"; \ 132 | else \ 133 | platform="linux/$(GOARCH)"; \ 134 | docker buildx build --progress=plain --platform=$$platform --output="type=docker,name=$$image" "$(SRCDIR)"; \ 135 | fi 136 | 137 | @echo "--- $@ ---" 138 | 139 | ci: deps generate test lint build 140 | -------------------------------------------------------------------------------- /build.go: -------------------------------------------------------------------------------- 1 | package gguf_packer 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/moby/buildkit/client/llb" 7 | "github.com/moby/buildkit/frontend/gateway/client" 8 | 9 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/builder" 10 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/ggufpackerfile2llb" 11 | ) 12 | 13 | // Build to build an image from a GGUFPackerfile. 14 | func Build(ctx context.Context, c client.Client) (*client.Result, error) { 15 | return builder.Build(ctx, c) 16 | } 17 | 18 | // ToLLB to convert a GGUFPackerfile to LLB. 19 | func ToLLB(ctx context.Context, bs []byte) (*llb.State, error) { 20 | st, _, _, _, err := ggufpackerfile2llb.ToLLB(ctx, bs, ggufpackerfile2llb.ConvertOpt{}) 21 | return st, err 22 | } 23 | -------------------------------------------------------------------------------- /buildkit/frontend/doc.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2024 Seal, Inc 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | // Package frontend is implemented in the BuildKit frontend, 5 | // most of the logic comes from the github.com/moby/buildkit/frontend. 6 | package frontend 7 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/builder/build.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "path" 8 | 9 | "github.com/containerd/platforms" 10 | ggufparser "github.com/gpustack/gguf-parser-go" 11 | "github.com/moby/buildkit/client/llb" 12 | "github.com/moby/buildkit/frontend" 13 | "github.com/moby/buildkit/frontend/gateway/client" 14 | "github.com/moby/buildkit/frontend/subrequests/lint" 15 | "github.com/moby/buildkit/frontend/subrequests/outline" 16 | "github.com/moby/buildkit/frontend/subrequests/targets" 17 | "github.com/moby/buildkit/solver/errdefs" 18 | "github.com/moby/buildkit/solver/pb" 19 | "github.com/pkg/errors" 20 | 21 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/ggufpackerfile2llb" 22 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/instructions" 23 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/linter" 24 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/parser" 25 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerui" 26 | specs "github.com/gpustack/gguf-packer-go/buildkit/frontend/specs/v1" 27 | ) 28 | 29 | func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { 30 | c = &withResolveCache{Client: c} 31 | bc, err := ggufpackerui.NewClient(c) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | src, err := bc.ReadEntrypoint(ctx, "GGUFPackerfile") 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | convertOpt := ggufpackerfile2llb.ConvertOpt{ 42 | Config: bc.Config, 43 | Client: bc, 44 | SourceMap: src.SourceMap, 45 | MetaResolver: c, 46 | Warn: func(rulename, description, url, msg string, location []parser.Range) { 47 | startLine := 0 48 | if len(location) > 0 { 49 | startLine = location[0].Start.Line 50 | } 51 | msg = linter.LintFormatShort(rulename, msg, startLine) 52 | src.Warn(ctx, msg, warnOpts(location, [][]byte{[]byte(description)}, url)) 53 | }, 54 | } 55 | 56 | if res, ok, err := bc.HandleSubrequest(ctx, ggufpackerui.RequestHandler{ 57 | Outline: func(ctx context.Context) (*outline.Outline, error) { 58 | return ggufpackerfile2llb.Outline(ctx, src.Data, convertOpt) 59 | }, 60 | ListTargets: func(ctx context.Context) (*targets.List, error) { 61 | return ggufpackerfile2llb.ListTargets(ctx, src.Data) 62 | }, 63 | Lint: func(ctx context.Context) (*lint.LintResults, error) { 64 | return ggufpackerfile2llb.Lint(ctx, src.Data, convertOpt) 65 | }, 66 | }); err != nil { 67 | return nil, err 68 | } else if ok { 69 | return res, nil 70 | } 71 | 72 | defer func() { 73 | var el *parser.ErrorLocation 74 | if errors.As(err, &el) { 75 | for _, l := range el.Locations { 76 | err = wrapSource(err, src.SourceMap, l) 77 | } 78 | } 79 | }() 80 | 81 | rb, err := bc.Build(ctx, func(ctx context.Context, platform *specs.Platform, idx int) (client.Reference, *specs.Image, *specs.Image, error) { 82 | opt := convertOpt 83 | opt.TargetPlatform = platform 84 | if idx != 0 { 85 | opt.Warn = nil 86 | } 87 | 88 | st, img, baseImg, pt, err := ggufpackerfile2llb.ToLLB(ctx, src.Data, opt) 89 | if err != nil { 90 | return nil, nil, nil, err 91 | } 92 | 93 | def, err := st.Marshal(ctx) 94 | if err != nil { 95 | return nil, nil, nil, errors.Wrapf(err, "failed to marshal LLB definition") 96 | } 97 | 98 | r, err := c.Solve(ctx, client.SolveRequest{ 99 | Definition: def.ToPB(), 100 | CacheImports: bc.CacheImports, 101 | }) 102 | if err != nil { 103 | return nil, nil, nil, errors.Wrapf(err, "failed to solve LLB definition") 104 | } 105 | 106 | ref, err := r.SingleRef() 107 | if err != nil { 108 | return nil, nil, nil, errors.Wrapf(err, "failed to get single ref") 109 | } 110 | 111 | if pt == nil { 112 | return ref, img, baseImg, nil 113 | } 114 | 115 | p := platforms.DefaultSpec() 116 | if platform != nil { 117 | p = *platform 118 | } 119 | id := platforms.Format(platforms.Normalize(p)) 120 | 121 | ps := []*instructions.CmdParameter{ 122 | pt.Cmd.Model, 123 | pt.Cmd.Drafter, 124 | pt.Cmd.Projector, 125 | } 126 | for i := range pt.Cmd.Adapters { 127 | ps = append(ps, &pt.Cmd.Adapters[i]) 128 | } 129 | 130 | img.Config.Size = 0 131 | for i := range ps { 132 | if ps[i] == nil { 133 | continue 134 | } 135 | 136 | runArgs := []string{ 137 | "gguf-parser", 138 | "--path", 139 | path.Join("/run/src", ps[i].Value), 140 | "--raw", 141 | "--raw-output", 142 | path.Join("/run/dest", ps[i].Type+".json"), 143 | } 144 | 145 | runOpt := []llb.RunOption{ 146 | llb.WithCustomName(fmt.Sprintf("[%s] parsing %s GGUF file", id, ps[i].Type)), 147 | ggufpackerfile2llb.Location(src.SourceMap, pt.Cmd.Location()), 148 | llb.Args(runArgs), 149 | llb.AddMount("/run/src", pt.State, llb.Readonly), 150 | llb.AddMount("/tmp", llb.Scratch(), llb.Tmpfs()), 151 | } 152 | if pt.IgnoreCache { 153 | runOpt = append(runOpt, llb.IgnoreCache) 154 | } 155 | run := llb.Image(bc.ParseImage).Run(runOpt...) 156 | 157 | pst := run.AddMount("/run/dest", llb.Scratch()) 158 | pdef, err := pst.Marshal(ctx) 159 | if err != nil { 160 | return nil, nil, nil, errors.Wrapf(err, "failed to marshal parsing LLB definition") 161 | } 162 | pr, err := c.Solve(ctx, frontend.SolveRequest{ 163 | Definition: pdef.ToPB(), 164 | }) 165 | if err != nil { 166 | return nil, nil, nil, errors.Wrapf(err, "failed to solve parsing LLB definition") 167 | } 168 | prr, err := pr.SingleRef() 169 | if err != nil { 170 | return nil, nil, nil, errors.Wrapf(err, "failed to get single parsing ref") 171 | } 172 | bs, err := prr.ReadFile(ctx, client.ReadRequest{ 173 | Filename: ps[i].Type + ".json", 174 | }) 175 | if err != nil { 176 | return nil, nil, nil, errors.Wrapf(err, "failed to read parsing result") 177 | } 178 | 179 | var gf ggufparser.GGUFFile 180 | if err = json.Unmarshal(bs, &gf); err != nil { 181 | return nil, nil, nil, errors.Wrapf(err, "failed to unmarshal parsing result") 182 | } 183 | m := gf.Metadata() 184 | mgf := specs.GGUFFile{ 185 | GGUFFile: gf, 186 | Architecture: m.Architecture, 187 | Parameters: m.Parameters, 188 | BitsPerWeight: m.BitsPerWeight, 189 | FileType: m.FileType, 190 | CmdParameterValue: ps[i].Value, 191 | CmdParameterIndex: ps[i].Index, 192 | } 193 | img.Config.Size += gf.Size 194 | switch ps[i].Type { 195 | case "model": 196 | // Labels. 197 | { 198 | if img.Config.Labels == nil { 199 | img.Config.Labels = map[string]string{} 200 | } 201 | lbs := img.Config.Labels 202 | setLabel(lbs, "gguf-packer", "gguf.model.vendor", "org.opencontainers.image.vendor") 203 | setLabel(lbs, "text-to-text", "gguf.model.usage") 204 | setLabel(lbs, m.Architecture, "gguf.model.architecture") 205 | setLabel(lbs, m.Parameters.String(), "gguf.model.parameters") 206 | setLabel(lbs, m.BitsPerWeight.String(), "gguf.model.bpw") 207 | setLabel(lbs, m.FileType.String(), "gguf.model.filetype") 208 | if v := m.Name; v != "" { 209 | setLabel(lbs, v, "gguf.model.name", "org.opencontainers.image.title") 210 | } 211 | if v := m.Author; v != "" { 212 | setLabel(lbs, v, "gguf.model.authors", "org.opencontainers.image.authors") 213 | } 214 | if v := m.URL; v != "" { 215 | setLabel(lbs, v, "gguf.model.url", "org.opencontainers.image.url") 216 | } 217 | if v := m.Description; v != "" { 218 | setLabel(lbs, v, "gguf.model.description", "org.opencontainers.image.description") 219 | } 220 | if v := m.License; v != "" { 221 | setLabel(lbs, v, "gguf.model.licenses", "org.opencontainers.image.licenses") 222 | } 223 | } 224 | img.Config.Model = &mgf 225 | case "drafter": 226 | img.Config.Drafter = &mgf 227 | case "projector": 228 | img.Config.Projector = &mgf 229 | case "adapter": 230 | img.Config.Adapters = append(img.Config.Adapters, &mgf) 231 | } 232 | } 233 | 234 | return ref, img, baseImg, nil 235 | }) 236 | if err != nil { 237 | return nil, err 238 | } 239 | 240 | return rb.Finalize() 241 | } 242 | 243 | func warnOpts(r []parser.Range, detail [][]byte, url string) client.WarnOpts { 244 | opts := client.WarnOpts{Level: 1, Detail: detail, URL: url} 245 | if r == nil { 246 | return opts 247 | } 248 | opts.Range = []*pb.Range{} 249 | for _, r := range r { 250 | opts.Range = append(opts.Range, &pb.Range{ 251 | Start: pb.Position{ 252 | Line: int32(r.Start.Line), 253 | Character: int32(r.Start.Character), 254 | }, 255 | End: pb.Position{ 256 | Line: int32(r.End.Line), 257 | Character: int32(r.End.Character), 258 | }, 259 | }) 260 | } 261 | return opts 262 | } 263 | 264 | func wrapSource(err error, sm *llb.SourceMap, ranges []parser.Range) error { 265 | if sm == nil { 266 | return err 267 | } 268 | s := errdefs.Source{ 269 | Info: &pb.SourceInfo{ 270 | Data: sm.Data, 271 | Filename: sm.Filename, 272 | Language: sm.Language, 273 | Definition: sm.Definition.ToPB(), 274 | }, 275 | Ranges: make([]*pb.Range, 0, len(ranges)), 276 | } 277 | for _, r := range ranges { 278 | s.Ranges = append(s.Ranges, &pb.Range{ 279 | Start: pb.Position{ 280 | Line: int32(r.Start.Line), 281 | Character: int32(r.Start.Character), 282 | }, 283 | End: pb.Position{ 284 | Line: int32(r.End.Line), 285 | Character: int32(r.End.Character), 286 | }, 287 | }) 288 | } 289 | return errdefs.WithSource(err, s) 290 | } 291 | 292 | func setLabel(lbs map[string]string, v string, k string, ks ...string) { 293 | if _, ok := lbs[k]; !ok { 294 | lbs[k] = v 295 | } 296 | for i := range ks { 297 | if _, ok := lbs[ks[i]]; !ok { 298 | lbs[ks[i]] = v 299 | } 300 | } 301 | } 302 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/builder/resolvecache.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/mitchellh/hashstructure/v2" 8 | "github.com/moby/buildkit/client/llb/sourceresolver" 9 | "github.com/moby/buildkit/frontend/gateway/client" 10 | "github.com/moby/buildkit/util/flightcontrol" 11 | "github.com/opencontainers/go-digest" 12 | ) 13 | 14 | // withResolveCache wraps client.Client so that ResolveImageConfig 15 | // calls are cached and deduplicated via flightcontrol.CachedGroup 16 | type withResolveCache struct { 17 | client.Client 18 | g flightcontrol.CachedGroup[*resolveResult] 19 | } 20 | 21 | type resolveResult struct { 22 | ref string 23 | dgst digest.Digest 24 | cfg []byte 25 | } 26 | 27 | var _ client.Client = &withResolveCache{} 28 | 29 | func (c *withResolveCache) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) { 30 | c.g.CacheError = true 31 | optHash, err := hashstructure.Hash(opt, hashstructure.FormatV2, nil) 32 | if err != nil { 33 | return "", "", nil, err 34 | } 35 | key := fmt.Sprintf("%s,%d", ref, optHash) 36 | res, err := c.g.Do(ctx, key, func(ctx context.Context) (*resolveResult, error) { 37 | ref, dgst, cfg, err := c.Client.ResolveImageConfig(ctx, ref, opt) 38 | if err != nil { 39 | return nil, err 40 | } 41 | return &resolveResult{ref, dgst, cfg}, nil 42 | }) 43 | if err != nil { 44 | return "", "", nil, err 45 | } 46 | return res.ref, res.dgst, res.cfg, nil 47 | } 48 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/command/command.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | // Define constants for the command strings 4 | const ( 5 | Add = "add" 6 | Arg = "arg" 7 | Cat = "cat" 8 | Cmd = "cmd" 9 | Copy = "copy" 10 | Convert = "convert" 11 | From = "from" 12 | Label = "label" 13 | Quantize = "quantize" 14 | ) 15 | 16 | // Commands is list of all GGUFPackerfile commands 17 | var Commands = map[string]struct{}{ 18 | Add: {}, 19 | Arg: {}, 20 | Cat: {}, 21 | Cmd: {}, 22 | Copy: {}, 23 | Convert: {}, 24 | From: {}, 25 | Label: {}, 26 | Quantize: {}, 27 | } 28 | 29 | func IsHeredocDirective(d string) bool { 30 | switch d { 31 | case Add, Copy, Cat: 32 | return true 33 | default: 34 | return false 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/ggufpackerfile2llb/image.go: -------------------------------------------------------------------------------- 1 | package ggufpackerfile2llb 2 | 3 | import ( 4 | specs "github.com/gpustack/gguf-packer-go/buildkit/frontend/specs/v1" 5 | ) 6 | 7 | func clone(src specs.Image) specs.Image { 8 | img := src 9 | img.Config = src.Config 10 | return img 11 | } 12 | 13 | func cloneX(src *specs.Image) *specs.Image { 14 | if src == nil { 15 | return nil 16 | } 17 | img := clone(*src) 18 | return &img 19 | } 20 | 21 | func emptyImage(platform specs.Platform) specs.Image { 22 | var img specs.Image 23 | img.Architecture = platform.Architecture 24 | img.OS = platform.OS 25 | img.OSVersion = platform.OSVersion 26 | if platform.OSFeatures != nil { 27 | img.OSFeatures = append([]string{}, platform.OSFeatures...) 28 | } 29 | img.Variant = platform.Variant 30 | return img 31 | } 32 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/ggufpackerfile2llb/outline.go: -------------------------------------------------------------------------------- 1 | package ggufpackerfile2llb 2 | 3 | import ( 4 | "maps" 5 | "sort" 6 | 7 | "github.com/moby/buildkit/frontend/subrequests/outline" 8 | pb "github.com/moby/buildkit/solver/pb" 9 | 10 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/instructions" 11 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/parser" 12 | ) 13 | 14 | type outlineCapture struct { 15 | allArgs map[string]argInfo 16 | usedArgs map[string]struct{} 17 | secrets map[string]secretInfo 18 | ssh map[string]sshInfo 19 | } 20 | 21 | type argInfo struct { 22 | value string 23 | definition instructions.KeyValuePairOptional 24 | deps map[string]struct{} 25 | location []parser.Range 26 | } 27 | 28 | type secretInfo struct { 29 | required bool 30 | location []parser.Range 31 | } 32 | 33 | type sshInfo struct { 34 | required bool 35 | location []parser.Range 36 | } 37 | 38 | func newOutlineCapture() outlineCapture { 39 | return outlineCapture{ 40 | allArgs: map[string]argInfo{}, 41 | usedArgs: map[string]struct{}{}, 42 | secrets: map[string]secretInfo{}, 43 | ssh: map[string]sshInfo{}, 44 | } 45 | } 46 | 47 | func (o outlineCapture) clone() outlineCapture { 48 | return outlineCapture{ 49 | allArgs: maps.Clone(o.allArgs), 50 | usedArgs: maps.Clone(o.usedArgs), 51 | secrets: maps.Clone(o.secrets), 52 | ssh: maps.Clone(o.ssh), 53 | } 54 | } 55 | 56 | func (o outlineCapture) markAllUsed(in map[string]struct{}) { 57 | for k := range in { 58 | if a, ok := o.allArgs[k]; ok { 59 | o.markAllUsed(a.deps) 60 | } 61 | o.usedArgs[k] = struct{}{} 62 | } 63 | } 64 | 65 | func (ds *dispatchState) args(visited map[string]struct{}) []outline.Arg { 66 | ds.outline.markAllUsed(ds.outline.usedArgs) 67 | 68 | args := make([]outline.Arg, 0, len(ds.outline.usedArgs)) 69 | for k := range ds.outline.usedArgs { 70 | if a, ok := ds.outline.allArgs[k]; ok { 71 | if _, ok := visited[k]; !ok { 72 | args = append(args, outline.Arg{ 73 | Name: a.definition.Key, 74 | Value: a.value, 75 | Description: a.definition.Comment, 76 | Location: toSourceLocation(a.location), 77 | }) 78 | visited[k] = struct{}{} 79 | } 80 | } 81 | } 82 | 83 | if ds.base != nil { 84 | args = append(args, ds.base.args(visited)...) 85 | } 86 | for d := range ds.deps { 87 | args = append(args, d.args(visited)...) 88 | } 89 | 90 | return args 91 | } 92 | 93 | func (ds *dispatchState) secrets(visited map[string]struct{}) []outline.Secret { 94 | secrets := make([]outline.Secret, 0, len(ds.outline.secrets)) 95 | for k, v := range ds.outline.secrets { 96 | if _, ok := visited[k]; !ok { 97 | secrets = append(secrets, outline.Secret{ 98 | Name: k, 99 | Required: v.required, 100 | Location: toSourceLocation(v.location), 101 | }) 102 | visited[k] = struct{}{} 103 | } 104 | } 105 | if ds.base != nil { 106 | secrets = append(secrets, ds.base.secrets(visited)...) 107 | } 108 | for d := range ds.deps { 109 | secrets = append(secrets, d.secrets(visited)...) 110 | } 111 | return secrets 112 | } 113 | 114 | func (ds *dispatchState) ssh(visited map[string]struct{}) []outline.SSH { 115 | ssh := make([]outline.SSH, 0, len(ds.outline.secrets)) 116 | for k, v := range ds.outline.ssh { 117 | if _, ok := visited[k]; !ok { 118 | ssh = append(ssh, outline.SSH{ 119 | Name: k, 120 | Required: v.required, 121 | Location: toSourceLocation(v.location), 122 | }) 123 | visited[k] = struct{}{} 124 | } 125 | } 126 | if ds.base != nil { 127 | ssh = append(ssh, ds.base.ssh(visited)...) 128 | } 129 | for d := range ds.deps { 130 | ssh = append(ssh, d.ssh(visited)...) 131 | } 132 | return ssh 133 | } 134 | 135 | func (ds *dispatchState) Outline(dt []byte) outline.Outline { 136 | args := ds.args(map[string]struct{}{}) 137 | sort.Slice(args, func(i, j int) bool { 138 | return compLocation(args[i].Location, args[j].Location) 139 | }) 140 | 141 | secrets := ds.secrets(map[string]struct{}{}) 142 | sort.Slice(secrets, func(i, j int) bool { 143 | return compLocation(secrets[i].Location, secrets[j].Location) 144 | }) 145 | 146 | ssh := ds.ssh(map[string]struct{}{}) 147 | sort.Slice(ssh, func(i, j int) bool { 148 | return compLocation(ssh[i].Location, ssh[j].Location) 149 | }) 150 | 151 | out := outline.Outline{ 152 | Name: ds.stage.Name, 153 | Description: ds.stage.Comment, 154 | Sources: [][]byte{dt}, 155 | Args: args, 156 | Secrets: secrets, 157 | SSH: ssh, 158 | } 159 | 160 | return out 161 | } 162 | 163 | func toSourceLocation(r []parser.Range) *pb.Location { 164 | if len(r) == 0 { 165 | return nil 166 | } 167 | arr := make([]*pb.Range, len(r)) 168 | for i, r := range r { 169 | arr[i] = &pb.Range{ 170 | Start: pb.Position{ 171 | Line: int32(r.Start.Line), 172 | Character: int32(r.Start.Character), 173 | }, 174 | End: pb.Position{ 175 | Line: int32(r.End.Line), 176 | Character: int32(r.End.Character), 177 | }, 178 | } 179 | } 180 | return &pb.Location{Ranges: arr} 181 | } 182 | 183 | func compLocation(a, b *pb.Location) bool { 184 | if a.SourceIndex != b.SourceIndex { 185 | return a.SourceIndex < b.SourceIndex 186 | } 187 | linea := 0 188 | lineb := 0 189 | if len(a.Ranges) > 0 { 190 | linea = int(a.Ranges[0].Start.Line) 191 | } 192 | if len(b.Ranges) > 0 { 193 | lineb = int(b.Ranges[0].Start.Line) 194 | } 195 | return linea < lineb 196 | } 197 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/ggufpackerfile2llb/platform.go: -------------------------------------------------------------------------------- 1 | package ggufpackerfile2llb 2 | 3 | import ( 4 | "github.com/containerd/platforms" 5 | 6 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/instructions" 7 | specs "github.com/gpustack/gguf-packer-go/buildkit/frontend/specs/v1" 8 | ) 9 | 10 | type platformOpt struct { 11 | targetPlatform specs.Platform 12 | buildPlatforms []specs.Platform 13 | implicitTarget bool 14 | } 15 | 16 | func buildPlatformOpt(opt *ConvertOpt) *platformOpt { 17 | buildPlatforms := opt.BuildPlatforms 18 | targetPlatform := opt.TargetPlatform 19 | implicitTargetPlatform := false 20 | 21 | if opt.TargetPlatform != nil && opt.BuildPlatforms == nil { 22 | buildPlatforms = []specs.Platform{*opt.TargetPlatform} 23 | } 24 | if len(buildPlatforms) == 0 { 25 | buildPlatforms = []specs.Platform{platforms.DefaultSpec()} 26 | } 27 | 28 | for i := range buildPlatforms { 29 | if buildPlatforms[i].OS == "darwin" { 30 | buildPlatforms[i].OS = "linux" 31 | } 32 | } 33 | 34 | if opt.TargetPlatform == nil { 35 | implicitTargetPlatform = true 36 | targetPlatform = &buildPlatforms[0] 37 | } 38 | 39 | return &platformOpt{ 40 | targetPlatform: *targetPlatform, 41 | buildPlatforms: buildPlatforms, 42 | implicitTarget: implicitTargetPlatform, 43 | } 44 | } 45 | 46 | func getPlatformArgs(po *platformOpt) []instructions.KeyValuePairOptional { 47 | bp := po.buildPlatforms[0] 48 | tp := po.targetPlatform 49 | m := map[string]string{ 50 | "BUILDPLATFORM": platforms.Format(bp), 51 | "BUILDOS": bp.OS, 52 | "BUILDARCH": bp.Architecture, 53 | "BUILDVARIANT": bp.Variant, 54 | "TARGETPLATFORM": platforms.Format(tp), 55 | "TARGETOS": tp.OS, 56 | "TARGETARCH": tp.Architecture, 57 | "TARGETVARIANT": tp.Variant, 58 | } 59 | opts := make([]instructions.KeyValuePairOptional, 0, len(m)) 60 | for k, v := range m { 61 | s := v 62 | opts = append(opts, instructions.KeyValuePairOptional{Key: k, Value: &s}) 63 | } 64 | return opts 65 | } 66 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/instructions/bflag.go: -------------------------------------------------------------------------------- 1 | package instructions 2 | 3 | import "github.com/moby/buildkit/frontend/dockerfile/instructions" 4 | 5 | type ( 6 | // BFlags contains all flags information for the builder 7 | BFlags = instructions.BFlags 8 | 9 | // Flag contains all information for a flag 10 | Flag = instructions.Flag 11 | ) 12 | 13 | // NewBFlagsWithArgs returns the new BFlags struct with Args set to args 14 | func NewBFlagsWithArgs(args []string) *BFlags { 15 | return instructions.NewBFlagsWithArgs(args) 16 | } 17 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/instructions/support.go: -------------------------------------------------------------------------------- 1 | package instructions 2 | 3 | import "strings" 4 | 5 | // handleJSONArgs parses command passed to CMD instruction in GGUFPackerfile 6 | // for exec form it returns untouched args slice 7 | // for shell form it returns concatenated args as the first element of a slice 8 | func handleJSONArgs(args []string, attributes map[string]bool) []string { 9 | if len(args) == 0 { 10 | return []string{} 11 | } 12 | 13 | if attributes != nil && attributes["json"] { 14 | return args 15 | } 16 | 17 | // literal string command, not an exec array 18 | return []string{strings.Join(args, " ")} 19 | } 20 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/linter/linter.go: -------------------------------------------------------------------------------- 1 | package linter 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/pkg/errors" 9 | 10 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/parser" 11 | ) 12 | 13 | type Config struct { 14 | Warn LintWarnFunc 15 | SkipRules []string 16 | SkipAll bool 17 | ReturnAsError bool 18 | } 19 | 20 | type Linter struct { 21 | SkippedRules map[string]struct{} 22 | CalledRules []string 23 | SkipAll bool 24 | ReturnAsError bool 25 | Warn LintWarnFunc 26 | } 27 | 28 | func New(config *Config) *Linter { 29 | toret := &Linter{ 30 | SkippedRules: map[string]struct{}{}, 31 | CalledRules: []string{}, 32 | Warn: config.Warn, 33 | } 34 | toret.SkipAll = config.SkipAll 35 | toret.ReturnAsError = config.ReturnAsError 36 | for _, rule := range config.SkipRules { 37 | toret.SkippedRules[rule] = struct{}{} 38 | } 39 | return toret 40 | } 41 | 42 | func (lc *Linter) Run(rule LinterRuleI, location []parser.Range, txt ...string) { 43 | if lc == nil || lc.Warn == nil || lc.SkipAll || rule.IsDeprecated() { 44 | return 45 | } 46 | rulename := rule.RuleName() 47 | if _, ok := lc.SkippedRules[rulename]; ok { 48 | return 49 | } 50 | lc.CalledRules = append(lc.CalledRules, rulename) 51 | rule.Run(lc.Warn, location, txt...) 52 | } 53 | 54 | func (lc *Linter) Error() error { 55 | if lc == nil || !lc.ReturnAsError { 56 | return nil 57 | } 58 | if len(lc.CalledRules) == 0 { 59 | return nil 60 | } 61 | var rules []string 62 | uniqueRules := map[string]struct{}{} 63 | for _, r := range lc.CalledRules { 64 | uniqueRules[r] = struct{}{} 65 | } 66 | for r := range uniqueRules { 67 | rules = append(rules, r) 68 | } 69 | return errors.Errorf("lint violation found for rules: %s", strings.Join(rules, ", ")) 70 | } 71 | 72 | type LinterRuleI interface { 73 | RuleName() string 74 | Run(warn LintWarnFunc, location []parser.Range, txt ...string) 75 | IsDeprecated() bool 76 | } 77 | 78 | type LinterRule[F any] struct { 79 | Name string 80 | Description string 81 | Deprecated bool 82 | URL string 83 | Format F 84 | } 85 | 86 | func (rule *LinterRule[F]) RuleName() string { 87 | return rule.Name 88 | } 89 | 90 | func (rule *LinterRule[F]) Run(warn LintWarnFunc, location []parser.Range, txt ...string) { 91 | if len(txt) == 0 { 92 | txt = []string{rule.Description} 93 | } 94 | short := strings.Join(txt, " ") 95 | warn(rule.Name, rule.Description, rule.URL, short, location) 96 | } 97 | 98 | func (rule *LinterRule[F]) IsDeprecated() bool { 99 | return rule.Deprecated 100 | } 101 | 102 | func LintFormatShort(rulename, msg string, line int) string { 103 | msg = fmt.Sprintf("%s: %s", rulename, msg) 104 | if line > 0 { 105 | msg = fmt.Sprintf("%s (line %d)", msg, line) 106 | } 107 | return msg 108 | } 109 | 110 | type LintWarnFunc func(rulename, description, url, fmtmsg string, location []parser.Range) 111 | 112 | func ParseLintOptions(checkStr string) (*Config, error) { 113 | checkStr = strings.TrimSpace(checkStr) 114 | if checkStr == "" { 115 | return &Config{}, nil 116 | } 117 | 118 | parts := strings.SplitN(checkStr, ";", 2) 119 | var skipSet []string 120 | var errorOnWarn, skipAll bool 121 | for _, p := range parts { 122 | k, v, ok := strings.Cut(p, "=") 123 | if !ok { 124 | return nil, errors.Errorf("invalid check option %q", p) 125 | } 126 | k = strings.TrimSpace(k) 127 | switch k { 128 | case "skip": 129 | v = strings.TrimSpace(v) 130 | if v == "all" { 131 | skipAll = true 132 | } else { 133 | skipSet = strings.Split(v, ",") 134 | for i, rule := range skipSet { 135 | skipSet[i] = strings.TrimSpace(rule) 136 | } 137 | } 138 | case "error": 139 | v, err := strconv.ParseBool(strings.TrimSpace(v)) 140 | if err != nil { 141 | return nil, errors.Wrapf(err, "failed to parse check option %q", p) 142 | } 143 | errorOnWarn = v 144 | default: 145 | return nil, errors.Errorf("invalid check option %q", k) 146 | } 147 | } 148 | return &Config{ 149 | SkipRules: skipSet, 150 | SkipAll: skipAll, 151 | ReturnAsError: errorOnWarn, 152 | }, nil 153 | } 154 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/linter/ruleset.go: -------------------------------------------------------------------------------- 1 | package linter 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | var ( 8 | RuleStageNameCasing = LinterRule[func(string) string]{ 9 | Name: "StageNameCasing", 10 | Description: "Stage names should be lowercase", 11 | URL: "https://docs.gpustack.ai/overview/", 12 | Format: func(stageName string) string { 13 | return fmt.Sprintf("Stage name '%s' should be lowercase", stageName) 14 | }, 15 | } 16 | RuleFromAsCasing = LinterRule[func(string, string) string]{ 17 | Name: "FromAsCasing", 18 | Description: "The 'as' keyword should match the case of the 'from' keyword", 19 | URL: "https://docs.gpustack.ai/overview/", 20 | Format: func(from, as string) string { 21 | return fmt.Sprintf("'%s' and '%s' keywords' casing do not match", as, from) 22 | }, 23 | } 24 | RuleNoEmptyContinuation = LinterRule[func() string]{ 25 | Name: "NoEmptyContinuation", 26 | Description: "Empty continuation lines will become errors in a future release", 27 | URL: "https://docs.gpustack.ai/overview/", 28 | Format: func() string { 29 | return "Empty continuation line" 30 | }, 31 | } 32 | RuleConsistentInstructionCasing = LinterRule[func(string, string) string]{ 33 | Name: "ConsistentInstructionCasing", 34 | Description: "All commands within the GGUFPackerfile should use the same casing (either upper or lower)", 35 | URL: "https://docs.gpustack.ai/overview/", 36 | Format: func(violatingCommand, correctCasing string) string { 37 | return fmt.Sprintf("Command '%s' should match the case of the command majority (%s)", violatingCommand, correctCasing) 38 | }, 39 | } 40 | RuleDuplicateStageName = LinterRule[func(string) string]{ 41 | Name: "DuplicateStageName", 42 | Description: "Stage names should be unique", 43 | URL: "https://docs.gpustack.ai/overview/", 44 | Format: func(stageName string) string { 45 | return fmt.Sprintf("Duplicate stage name %q, stage names should be unique", stageName) 46 | }, 47 | } 48 | RuleReservedStageName = LinterRule[func(string) string]{ 49 | Name: "ReservedStageName", 50 | Description: "Reserved words should not be used as stage names", 51 | URL: "https://docs.gpustack.ai/overview/", 52 | Format: func(reservedStageName string) string { 53 | return fmt.Sprintf("Stage name should not use the same name as reserved stage %q", reservedStageName) 54 | }, 55 | } 56 | RuleUndefinedArgInFrom = LinterRule[func(string, string) string]{ 57 | Name: "UndefinedArgInFrom", 58 | Description: "FROM command must use declared ARGs", 59 | URL: "https://docs.gpustack.ai/overview/", 60 | Format: func(baseArg, suggest string) string { 61 | out := fmt.Sprintf("FROM argument '%s' is not declared", baseArg) 62 | if suggest != "" { 63 | out += fmt.Sprintf(" (did you mean %s?)", suggest) 64 | } 65 | return out 66 | }, 67 | } 68 | RuleUndefinedVar = LinterRule[func(string, string) string]{ 69 | Name: "UndefinedVar", 70 | Description: "Variables should be defined before their use", 71 | URL: "https://docs.gpustack.ai/overview/", 72 | Format: func(arg, suggest string) string { 73 | out := fmt.Sprintf("Usage of undefined variable '$%s'", arg) 74 | if suggest != "" { 75 | out += fmt.Sprintf(" (did you mean $%s?)", suggest) 76 | } 77 | return out 78 | }, 79 | } 80 | RuleMultipleInstructionsDisallowed = LinterRule[func(instructionName string) string]{ 81 | Name: "MultipleInstructionsDisallowed", 82 | Description: "Multiple instructions of the same type should not be used in the same stage", 83 | URL: "https://docs.gpustack.ai/overview/", 84 | Format: func(instructionName string) string { 85 | return fmt.Sprintf("Multiple %s instructions should not be used in the same stage because only the last one will be used", instructionName) 86 | }, 87 | } 88 | RuleLegacyKeyValueFormat = LinterRule[func(cmdName string) string]{ 89 | Name: "LegacyKeyValueFormat", 90 | Description: "Legacy key/value format with whitespace separator should not be used", 91 | URL: "https://docs.gpustack.ai/overview/", 92 | Format: func(cmdName string) string { 93 | return fmt.Sprintf("\"%s key=value\" should be used instead of legacy \"%s key value\" format", cmdName, cmdName) 94 | }, 95 | } 96 | RuleInvalidBaseImagePlatform = LinterRule[func(string, string, string) string]{ 97 | Name: "InvalidBaseImagePlatform", 98 | Description: "Base image platform does not match expected target platform", 99 | URL: "https://docs.gpustack.ai/overview/", 100 | Format: func(image, expected, actual string) string { 101 | return fmt.Sprintf("Base image %s was pulled with platform %q, expected %q for current build", image, actual, expected) 102 | }, 103 | } 104 | RuleSecretsUsedInArgOrEnv = LinterRule[func(string, string) string]{ 105 | Name: "SecretsUsedInArgOrEnv", 106 | Description: "Sensitive data should not be used in the ARG commands", 107 | URL: "https://docs.gpustack.ai/overview/", 108 | Format: func(instruction, secretKey string) string { 109 | return fmt.Sprintf("Do not use ARG or ENV instructions for sensitive data (%s %q)", instruction, secretKey) 110 | }, 111 | } 112 | RuleInvalidDefaultArgInFrom = LinterRule[func(string) string]{ 113 | Name: "InvalidDefaultArgInFrom", 114 | Description: "Default value for global ARG results in an empty or invalid base image name", 115 | URL: "https://docs.gpustack.ai/overview/", 116 | Format: func(baseName string) string { 117 | return fmt.Sprintf("Default value for ARG %v results in empty or invalid base image name", baseName) 118 | }, 119 | } 120 | RuleCopyIgnoredFile = LinterRule[func(string, string) string]{ 121 | Name: "CopyIgnoredFile", 122 | Description: "Attempting to Copy file that is excluded by .ggufpackerignore", 123 | URL: "https://docs.gpustack.ai/overview/", 124 | Format: func(cmd, file string) string { 125 | return fmt.Sprintf("Attempting to %s file %q that is excluded by .ggufpackerignore", cmd, file) 126 | }, 127 | } 128 | ) 129 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/parser/directives.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/json" 7 | "fmt" 8 | "regexp" 9 | "strings" 10 | 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | const ( 15 | keySyntax = "syntax" 16 | keyCheck = "check" 17 | keyEscape = "escape" 18 | ) 19 | 20 | var validDirectives = map[string]struct{}{ 21 | keySyntax: {}, 22 | keyEscape: {}, 23 | keyCheck: {}, 24 | } 25 | 26 | type Directive struct { 27 | Name string 28 | Value string 29 | Location []Range 30 | } 31 | 32 | // DirectiveParser is a parser for GGUFPackerfile directives that enforces the 33 | // quirks of the directive parser. 34 | type DirectiveParser struct { 35 | line int 36 | regexp *regexp.Regexp 37 | seen map[string]struct{} 38 | done bool 39 | } 40 | 41 | func (d *DirectiveParser) setComment(comment string) { 42 | d.regexp = regexp.MustCompile(fmt.Sprintf(`^%s\s*([a-zA-Z][a-zA-Z0-9]*)\s*=\s*(.+?)\s*$`, comment)) 43 | } 44 | 45 | func (d *DirectiveParser) ParseLine(line []byte) (*Directive, error) { 46 | d.line++ 47 | if d.done { 48 | return nil, nil 49 | } 50 | if d.regexp == nil { 51 | d.setComment("#") 52 | } 53 | 54 | match := d.regexp.FindSubmatch(line) 55 | if len(match) == 0 { 56 | d.done = true 57 | return nil, nil 58 | } 59 | 60 | k := strings.ToLower(string(match[1])) 61 | if _, ok := validDirectives[k]; !ok { 62 | d.done = true 63 | return nil, nil 64 | } 65 | if d.seen == nil { 66 | d.seen = map[string]struct{}{} 67 | } 68 | if _, ok := d.seen[k]; ok { 69 | return nil, errors.Errorf("only one %s parser directive can be used", k) 70 | } 71 | d.seen[k] = struct{}{} 72 | 73 | v := string(match[2]) 74 | 75 | directive := Directive{ 76 | Name: k, 77 | Value: v, 78 | Location: []Range{{ 79 | Start: Position{Line: d.line}, 80 | End: Position{Line: d.line}, 81 | }}, 82 | } 83 | return &directive, nil 84 | } 85 | 86 | func (d *DirectiveParser) ParseAll(data []byte) ([]*Directive, error) { 87 | scanner := bufio.NewScanner(bytes.NewReader(data)) 88 | var directives []*Directive 89 | for scanner.Scan() { 90 | if d.done { 91 | break 92 | } 93 | 94 | d, err := d.ParseLine(scanner.Bytes()) 95 | if err != nil { 96 | return directives, err 97 | } 98 | if d != nil { 99 | directives = append(directives, d) 100 | } 101 | } 102 | return directives, nil 103 | } 104 | 105 | // DetectSyntax returns the syntax of provided input. 106 | // 107 | // The traditional dockerfile directives '# syntax = ...' are used by default, 108 | // however, the function will also fallback to c-style directives '// syntax = ...' 109 | // and json-encoded directives '{ "syntax": "..." }'. Finally, starting lines 110 | // with '#!' are treated as shebangs and ignored. 111 | // 112 | // This allows for a flexible range of input formats, and appropriate syntax 113 | // selection. 114 | func DetectSyntax(dt []byte) (string, string, []Range, bool) { 115 | return ParseDirective(keySyntax, dt) 116 | } 117 | 118 | func ParseDirective(key string, dt []byte) (string, string, []Range, bool) { 119 | dt, hadShebang, err := discardShebang(dt) 120 | if err != nil { 121 | return "", "", nil, false 122 | } 123 | line := 0 124 | if hadShebang { 125 | line++ 126 | } 127 | 128 | // use default directive parser, and search for #key= 129 | directiveParser := DirectiveParser{line: line} 130 | if syntax, cmdline, loc, ok := detectDirectiveFromParser(key, dt, directiveParser); ok { 131 | return syntax, cmdline, loc, true 132 | } 133 | 134 | // use directive with different comment prefix, and search for //key= 135 | directiveParser = DirectiveParser{line: line} 136 | directiveParser.setComment("//") 137 | if syntax, cmdline, loc, ok := detectDirectiveFromParser(key, dt, directiveParser); ok { 138 | return syntax, cmdline, loc, true 139 | } 140 | 141 | // use json directive, and search for { "key": "..." } 142 | jsonDirective := map[string]string{} 143 | if err := json.Unmarshal(dt, &jsonDirective); err == nil { 144 | if v, ok := jsonDirective[key]; ok { 145 | loc := []Range{{ 146 | Start: Position{Line: line}, 147 | End: Position{Line: line}, 148 | }} 149 | return v, v, loc, true 150 | } 151 | } 152 | 153 | return "", "", nil, false 154 | } 155 | 156 | func detectDirectiveFromParser(key string, dt []byte, parser DirectiveParser) (string, string, []Range, bool) { 157 | directives, _ := parser.ParseAll(dt) 158 | for _, d := range directives { 159 | if d.Name == key { 160 | p, _, _ := strings.Cut(d.Value, " ") 161 | return p, d.Value, d.Location, true 162 | } 163 | } 164 | return "", "", nil, false 165 | } 166 | 167 | func discardShebang(dt []byte) ([]byte, bool, error) { 168 | line, rest, _ := bytes.Cut(dt, []byte("\n")) 169 | if bytes.HasPrefix(line, []byte("#!")) { 170 | return rest, true, nil 171 | } 172 | return dt, false, nil 173 | } 174 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/parser/errors.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/moby/buildkit/frontend/dockerfile/parser" 5 | ) 6 | 7 | type ( 8 | // ErrorLocation gives a location in source code that caused the error 9 | ErrorLocation = parser.ErrorLocation 10 | 11 | // Range is a code section between two positions 12 | Range = parser.Range 13 | 14 | // Position is a point in source code 15 | Position = parser.Position 16 | ) 17 | 18 | // WithLocation extends an error with a source code location 19 | func WithLocation(err error, location []Range) error { 20 | return parser.WithLocation(err, location) 21 | } 22 | 23 | func withLocation(err error, start, end int) error { 24 | return WithLocation(err, toRanges(start, end)) 25 | } 26 | 27 | func toRanges(start, end int) (r []Range) { 28 | if end <= start { 29 | end = start 30 | } 31 | for i := start; i <= end; i++ { 32 | r = append(r, Range{Start: Position{Line: i}, End: Position{Line: i}}) 33 | } 34 | return 35 | } 36 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/parser/line_parsers.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "unicode" 7 | "unicode/utf8" 8 | 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | var ( 13 | errGGUFPackerfileNotStringArray = errors.New("when using JSON array syntax, arrays must be comprised of strings only") 14 | errGGUFPackerfileNotJSONArray = errors.New("not a JSON array") 15 | ) 16 | 17 | // ignore the current argument. This will still leave a command parsed, but 18 | // will not incorporate the arguments into the ast. 19 | func parseIgnore(rest string, d *directives) (*Node, map[string]bool, error) { 20 | return &Node{}, nil, nil 21 | } 22 | 23 | // helper to parse words (i.e space delimited or quoted strings) in a statement. 24 | // The quotes are preserved as part of this function and they are stripped later 25 | // as part of processWords(). 26 | func parseWords(rest string, d *directives) []string { 27 | const ( 28 | inSpaces = iota // looking for start of a word 29 | inWord 30 | inQuote 31 | ) 32 | 33 | var ( 34 | words []string 35 | phase = inSpaces 36 | quote = '\000' 37 | blankOK bool 38 | ch rune 39 | chWidth int 40 | sbuilder strings.Builder 41 | ) 42 | 43 | for pos := 0; pos <= len(rest); pos += chWidth { 44 | if pos != len(rest) { 45 | ch, chWidth = utf8.DecodeRuneInString(rest[pos:]) 46 | } 47 | 48 | if phase == inSpaces { // Looking for start of word 49 | if pos == len(rest) { // end of input 50 | break 51 | } 52 | if unicode.IsSpace(ch) { // skip spaces 53 | continue 54 | } 55 | phase = inWord // found it, fall through 56 | } 57 | 58 | if (phase == inWord) && (pos == len(rest)) { 59 | if blankOK || sbuilder.Len() > 0 { 60 | words = append(words, sbuilder.String()) 61 | } 62 | break 63 | } 64 | 65 | if phase == inWord { 66 | if unicode.IsSpace(ch) { 67 | phase = inSpaces 68 | if blankOK || sbuilder.Len() > 0 { 69 | words = append(words, sbuilder.String()) 70 | } 71 | sbuilder.Reset() 72 | blankOK = false 73 | continue 74 | } 75 | if ch == '\'' || ch == '"' { 76 | quote = ch 77 | blankOK = true 78 | phase = inQuote 79 | } 80 | if ch == d.escapeToken { 81 | if pos+chWidth == len(rest) { 82 | continue // just skip an escape token at end of line 83 | } 84 | // If we're not quoted and we see an escape token, then always just 85 | // add the escape token plus the char to the word, even if the char 86 | // is a quote. 87 | sbuilder.WriteRune(ch) 88 | pos += chWidth 89 | ch, chWidth = utf8.DecodeRuneInString(rest[pos:]) 90 | } 91 | sbuilder.WriteRune(ch) 92 | 93 | continue 94 | } 95 | 96 | if ch == quote { 97 | phase = inWord 98 | } 99 | // The escape token is special except for ' quotes - can't escape anything for ' 100 | if ch == d.escapeToken && quote != '\'' { 101 | if pos+chWidth == len(rest) { 102 | phase = inWord 103 | continue // just skip the escape token at end 104 | } 105 | pos += chWidth 106 | sbuilder.WriteRune(ch) 107 | ch, chWidth = utf8.DecodeRuneInString(rest[pos:]) 108 | } 109 | sbuilder.WriteRune(ch) 110 | } 111 | 112 | return words 113 | } 114 | 115 | // parse environment like statements. Note that this does *not* handle 116 | // variable interpolation, which will be handled in the evaluator. 117 | func parseNameVal(rest string, key string, d *directives) (*Node, error) { 118 | // This is kind of tricky because we need to support the old 119 | // variant: KEY name value 120 | // as well as the new one: KEY name=value ... 121 | // The trigger to know which one is being used will be whether we hit 122 | // a space or = first. space ==> old, "=" ==> new 123 | 124 | words := parseWords(rest, d) 125 | if len(words) == 0 { 126 | return nil, nil 127 | } 128 | 129 | // Old format (KEY name value) 130 | if !strings.Contains(words[0], "=") { 131 | parts := reWhitespace.Split(rest, 2) 132 | if len(parts) < 2 { 133 | return nil, errors.Errorf("%s must have two arguments", key) 134 | } 135 | return newKeyValueNode(parts[0], parts[1], ""), nil 136 | } 137 | 138 | var rootNode *Node 139 | var prevNode *Node 140 | for _, word := range words { 141 | if !strings.Contains(word, "=") { 142 | return nil, errors.Errorf("Syntax error - can't find = in %q. Must be of the form: name=value", word) 143 | } 144 | 145 | parts := strings.SplitN(word, "=", 2) 146 | node := newKeyValueNode(parts[0], parts[1], "=") 147 | rootNode, prevNode = appendKeyValueNode(node, rootNode, prevNode) 148 | } 149 | 150 | return rootNode, nil 151 | } 152 | 153 | func newKeyValueNode(key, value, sep string) *Node { 154 | return &Node{ 155 | Value: key, 156 | Next: &Node{ 157 | Value: value, 158 | Next: &Node{Value: sep}, 159 | }, 160 | } 161 | } 162 | 163 | func appendKeyValueNode(node, rootNode, prevNode *Node) (*Node, *Node) { 164 | if rootNode == nil { 165 | rootNode = node 166 | } 167 | if prevNode != nil { 168 | prevNode.Next = node 169 | } 170 | 171 | for prevNode = node.Next; prevNode.Next != nil; { 172 | prevNode = prevNode.Next 173 | } 174 | return rootNode, prevNode 175 | } 176 | 177 | func parseLabel(rest string, d *directives) (*Node, map[string]bool, error) { 178 | node, err := parseNameVal(rest, "LABEL", d) 179 | return node, nil, err 180 | } 181 | 182 | // parses a statement containing one or more keyword definition(s) and/or 183 | // value assignments, like `name1 name2= name3="" name4=value`. 184 | // Note that this is a stricter format than the old format of assignment, 185 | // allowed by parseNameVal(), in a way that this only allows assignment of the 186 | // form `keyword=[]` like `name2=`, `name3=""`, and `name4=value` above. 187 | // In addition, a keyword definition alone is of the form `keyword` like `name1` 188 | // above. And the assignments `name2=` and `name3=""` are equivalent and 189 | // assign an empty value to the respective keywords. 190 | func parseNameOrNameVal(rest string, d *directives) (*Node, map[string]bool, error) { 191 | words := parseWords(rest, d) 192 | if len(words) == 0 { 193 | return nil, nil, nil 194 | } 195 | 196 | var ( 197 | rootnode *Node 198 | prevNode *Node 199 | ) 200 | for i, word := range words { 201 | node := &Node{} 202 | node.Value = word 203 | if i == 0 { 204 | rootnode = node 205 | } else { 206 | prevNode.Next = node 207 | } 208 | prevNode = node 209 | } 210 | 211 | return rootnode, nil, nil 212 | } 213 | 214 | // parses a whitespace-delimited set of arguments. The result is effectively a 215 | // linked list of string arguments. 216 | func parseStringsWhitespaceDelimited(rest string, d *directives) (*Node, map[string]bool, error) { 217 | if rest == "" { 218 | return nil, nil, nil 219 | } 220 | 221 | node := &Node{} 222 | rootnode := node 223 | prevnode := node 224 | for _, str := range reWhitespace.Split(rest, -1) { // use regexp 225 | prevnode = node 226 | node.Value = str 227 | node.Next = &Node{} 228 | node = node.Next 229 | } 230 | 231 | // XXX to get around regexp.Split *always* providing an empty string at the 232 | // end due to how our loop is constructed, nil out the last node in the 233 | // chain. 234 | prevnode.Next = nil 235 | 236 | return rootnode, nil, nil 237 | } 238 | 239 | // parseJSON converts JSON arrays to an AST. 240 | func parseJSON(rest string) (*Node, map[string]bool, error) { 241 | rest = strings.TrimLeftFunc(rest, unicode.IsSpace) 242 | if !strings.HasPrefix(rest, "[") { 243 | return nil, nil, errGGUFPackerfileNotJSONArray 244 | } 245 | 246 | var myJSON []interface{} 247 | if err := json.Unmarshal([]byte(rest), &myJSON); err != nil { 248 | return nil, nil, err 249 | } 250 | 251 | var top, prev *Node 252 | for _, str := range myJSON { 253 | s, ok := str.(string) 254 | if !ok { 255 | return nil, nil, errGGUFPackerfileNotStringArray 256 | } 257 | 258 | node := &Node{Value: s} 259 | if prev == nil { 260 | top = node 261 | } else { 262 | prev.Next = node 263 | } 264 | prev = node 265 | } 266 | 267 | return top, map[string]bool{"json": true}, nil 268 | } 269 | 270 | // parseMaybeJSON determines if the argument appears to be a JSON array. If 271 | // so, passes to parseJSON; if not, quotes the result and returns a single 272 | // node. 273 | func parseMaybeJSON(rest string, d *directives) (*Node, map[string]bool, error) { 274 | if rest == "" { 275 | return nil, nil, nil 276 | } 277 | 278 | node, attrs, err := parseJSON(rest) 279 | 280 | if err == nil { 281 | return node, attrs, nil 282 | } 283 | if errors.Is(err, errGGUFPackerfileNotStringArray) { 284 | return nil, nil, err 285 | } 286 | 287 | node = &Node{} 288 | node.Value = rest 289 | return node, nil, nil 290 | } 291 | 292 | // parseMaybeJSONToList determines if the argument appears to be a JSON array. If 293 | // so, passes to parseJSON; if not, attempts to parse it as a whitespace 294 | // delimited string. 295 | func parseMaybeJSONToList(rest string, d *directives) (*Node, map[string]bool, error) { 296 | node, attrs, err := parseJSON(rest) 297 | 298 | if err == nil { 299 | return node, attrs, nil 300 | } 301 | if errors.Is(err, errGGUFPackerfileNotStringArray) { 302 | return nil, nil, err 303 | } 304 | 305 | return parseStringsWhitespaceDelimited(rest, d) 306 | } 307 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerfile/parser/split_command.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "strings" 5 | "unicode" 6 | ) 7 | 8 | // splitCommand takes a single line of text and parses out the cmd and args, 9 | // which are used for dispatching to more exact parsing functions. 10 | func splitCommand(line string) (string, []string, string, error) { 11 | var args string 12 | var flags []string 13 | 14 | // Make sure we get the same results irrespective of leading/trailing spaces 15 | cmdline := reWhitespace.Split(strings.TrimSpace(line), 2) 16 | 17 | if len(cmdline) == 2 { 18 | var err error 19 | args, flags, err = extractBuilderFlags(cmdline[1]) 20 | if err != nil { 21 | return "", nil, "", err 22 | } 23 | } 24 | 25 | return cmdline[0], flags, strings.TrimSpace(args), nil 26 | } 27 | 28 | func extractBuilderFlags(line string) (string, []string, error) { 29 | const ( 30 | inSpaces = iota // looking for start of a word 31 | inWord 32 | inQuote 33 | ) 34 | 35 | var ( 36 | words []string 37 | phase = inSpaces 38 | quote = '\000' 39 | blankOK bool 40 | ch rune 41 | sbuilder strings.Builder 42 | ) 43 | 44 | for pos := 0; pos <= len(line); pos++ { 45 | if pos != len(line) { 46 | ch = rune(line[pos]) 47 | } 48 | 49 | if phase == inSpaces { // Looking for start of word 50 | if pos == len(line) { // end of input 51 | break 52 | } 53 | if unicode.IsSpace(ch) { // skip spaces 54 | continue 55 | } 56 | 57 | // Only keep going if the next word starts with -- 58 | if ch != '-' || pos+1 == len(line) || rune(line[pos+1]) != '-' { 59 | return line[pos:], words, nil 60 | } 61 | 62 | phase = inWord // found something with "--", fall through 63 | } 64 | 65 | if (phase == inWord) && (pos == len(line)) { 66 | if word := sbuilder.String(); word != "--" && (blankOK || len(word) > 0) { 67 | words = append(words, word) 68 | } 69 | break 70 | } 71 | 72 | if phase == inWord { 73 | if unicode.IsSpace(ch) { 74 | word := sbuilder.String() 75 | phase = inSpaces 76 | if word == "--" { 77 | return line[pos:], words, nil 78 | } 79 | if blankOK || len(word) > 0 { 80 | words = append(words, word) 81 | } 82 | sbuilder.Reset() 83 | blankOK = false 84 | continue 85 | } 86 | if ch == '\'' || ch == '"' { 87 | quote = ch 88 | blankOK = true 89 | phase = inQuote 90 | continue 91 | } 92 | if ch == '\\' { 93 | if pos+1 == len(line) { 94 | continue // just skip \ at end 95 | } 96 | pos++ 97 | ch = rune(line[pos]) 98 | } 99 | if _, err := sbuilder.WriteRune(ch); err != nil { 100 | return "", nil, err 101 | } 102 | continue 103 | } 104 | 105 | if ch == quote { 106 | phase = inWord 107 | continue 108 | } 109 | if ch == '\\' { 110 | if pos+1 == len(line) { 111 | phase = inWord 112 | continue // just skip \ at end 113 | } 114 | pos++ 115 | ch = rune(line[pos]) 116 | } 117 | if _, err := sbuilder.WriteRune(ch); err != nil { 118 | return "", nil, err 119 | } 120 | } 121 | 122 | return "", words, nil 123 | } 124 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerui/attr.go: -------------------------------------------------------------------------------- 1 | package ggufpackerui 2 | 3 | import ( 4 | "net" 5 | "strconv" 6 | "strings" 7 | "time" 8 | 9 | "github.com/containerd/platforms" 10 | "github.com/docker/go-units" 11 | "github.com/moby/buildkit/client/llb" 12 | "github.com/moby/buildkit/solver/pb" 13 | "github.com/pkg/errors" 14 | "github.com/tonistiigi/go-csvvalue" 15 | 16 | specs "github.com/gpustack/gguf-packer-go/buildkit/frontend/specs/v1" 17 | ) 18 | 19 | func parsePlatforms(v string) ([]specs.Platform, error) { 20 | var pp []specs.Platform 21 | for _, v := range strings.Split(v, ",") { 22 | p, err := platforms.Parse(v) 23 | if err != nil { 24 | return nil, errors.Wrapf(err, "failed to parse target platform %s", v) 25 | } 26 | pp = append(pp, platforms.Normalize(p)) 27 | } 28 | return pp, nil 29 | } 30 | 31 | func parseResolveMode(v string) (llb.ResolveMode, error) { 32 | switch v { 33 | case pb.AttrImageResolveModeDefault, "": 34 | return llb.ResolveModeDefault, nil 35 | case pb.AttrImageResolveModeForcePull: 36 | return llb.ResolveModeForcePull, nil 37 | case pb.AttrImageResolveModePreferLocal: 38 | return llb.ResolveModePreferLocal, nil 39 | default: 40 | return 0, errors.Errorf("invalid image-resolve-mode: %s", v) 41 | } 42 | } 43 | 44 | func parseExtraHosts(v string) ([]llb.HostIP, error) { 45 | if v == "" { 46 | return nil, nil 47 | } 48 | out := make([]llb.HostIP, 0) 49 | fields, err := csvvalue.Fields(v, nil) 50 | if err != nil { 51 | return nil, err 52 | } 53 | for _, field := range fields { 54 | key, val, ok := strings.Cut(strings.ToLower(field), "=") 55 | if !ok { 56 | return nil, errors.Errorf("invalid key-value pair %s", field) 57 | } 58 | ip := net.ParseIP(val) 59 | if ip == nil { 60 | return nil, errors.Errorf("failed to parse IP %s", val) 61 | } 62 | out = append(out, llb.HostIP{Host: key, IP: ip}) 63 | } 64 | return out, nil 65 | } 66 | 67 | func parseShmSize(v string) (int64, error) { 68 | if len(v) == 0 { 69 | return 0, nil 70 | } 71 | kb, err := strconv.ParseInt(v, 10, 64) 72 | if err != nil { 73 | return 0, err 74 | } 75 | return kb, nil 76 | } 77 | 78 | func parseUlimits(v string) ([]pb.Ulimit, error) { 79 | if v == "" { 80 | return nil, nil 81 | } 82 | out := make([]pb.Ulimit, 0) 83 | fields, err := csvvalue.Fields(v, nil) 84 | if err != nil { 85 | return nil, err 86 | } 87 | for _, field := range fields { 88 | ulimit, err := units.ParseUlimit(field) 89 | if err != nil { 90 | return nil, err 91 | } 92 | out = append(out, pb.Ulimit{ 93 | Name: ulimit.Name, 94 | Soft: ulimit.Soft, 95 | Hard: ulimit.Hard, 96 | }) 97 | } 98 | return out, nil 99 | } 100 | 101 | func parseNetMode(v string) (pb.NetMode, error) { 102 | if v == "" { 103 | return llb.NetModeSandbox, nil 104 | } 105 | switch v { 106 | case "none": 107 | return llb.NetModeNone, nil 108 | case "host": 109 | return llb.NetModeHost, nil 110 | case "sandbox": 111 | return llb.NetModeSandbox, nil 112 | default: 113 | return 0, errors.Errorf("invalid netmode %s", v) 114 | } 115 | } 116 | 117 | func parseSourceDateEpoch(v string) (*time.Time, error) { 118 | if v == "" { 119 | return nil, nil 120 | } 121 | sde, err := strconv.ParseInt(v, 10, 64) 122 | if err != nil { 123 | return nil, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH: %s", v) 124 | } 125 | tm := time.Unix(sde, 0).UTC() 126 | return &tm, nil 127 | } 128 | 129 | func filter(opt map[string]string, key string) map[string]string { 130 | m := map[string]string{} 131 | for k, v := range opt { 132 | if strings.HasPrefix(k, key) { 133 | m[strings.TrimPrefix(k, key)] = v 134 | } 135 | } 136 | return m 137 | } 138 | 139 | func tenary[T any](c bool, t, f T) T { 140 | if c { 141 | return t 142 | } 143 | return f 144 | } 145 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerui/build.go: -------------------------------------------------------------------------------- 1 | package ggufpackerui 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/containerd/platforms" 9 | "github.com/moby/buildkit/exporter/containerimage/exptypes" 10 | "github.com/moby/buildkit/frontend/gateway/client" 11 | "github.com/pkg/errors" 12 | "golang.org/x/sync/errgroup" 13 | 14 | specs "github.com/gpustack/gguf-packer-go/buildkit/frontend/specs/v1" 15 | ) 16 | 17 | type BuildFunc func(ctx context.Context, platform *specs.Platform, idx int) (r client.Reference, img, baseImg *specs.Image, err error) 18 | 19 | func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, error) { 20 | res := client.NewResult() 21 | 22 | targets := make([]*specs.Platform, 0, len(bc.TargetPlatforms)) 23 | for _, p := range bc.TargetPlatforms { 24 | p := p 25 | targets = append(targets, &p) 26 | } 27 | if len(targets) == 0 { 28 | targets = append(targets, nil) 29 | } 30 | expPlatforms := &exptypes.Platforms{ 31 | Platforms: make([]exptypes.Platform, len(targets)), 32 | } 33 | 34 | eg, ctx := errgroup.WithContext(ctx) 35 | 36 | for i, tp := range targets { 37 | i, tp := i, tp 38 | eg.Go(func() error { 39 | ref, img, baseImg, err := fn(ctx, tp, i) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | config, err := json.Marshal(img) 45 | if err != nil { 46 | return errors.Wrapf(err, "failed to marshal image config") 47 | } 48 | 49 | var baseConfig []byte 50 | if baseImg != nil { 51 | baseConfig, err = json.Marshal(baseImg) 52 | if err != nil { 53 | return errors.Wrapf(err, "failed to marshal source image config") 54 | } 55 | } 56 | 57 | p := platforms.DefaultSpec() 58 | if tp != nil { 59 | p = *tp 60 | } 61 | 62 | // in certain conditions we allow input platform to be extended from base image 63 | if p.OS == "windows" && img.OS == p.OS { 64 | if p.OSVersion == "" && img.OSVersion != "" { 65 | p.OSVersion = img.OSVersion 66 | } 67 | if p.OSFeatures == nil && len(img.OSFeatures) > 0 { 68 | p.OSFeatures = append([]string{}, img.OSFeatures...) 69 | } 70 | } 71 | 72 | p = platforms.Normalize(p) 73 | k := platforms.Format(p) 74 | 75 | if bc.MultiPlatformRequested { 76 | res.AddRef(k, ref) 77 | res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterImageConfigKey, k), config) 78 | if len(baseConfig) > 0 { 79 | res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterImageBaseConfigKey, k), baseConfig) 80 | } 81 | } else { 82 | res.SetRef(ref) 83 | res.AddMeta(exptypes.ExporterImageConfigKey, config) 84 | if len(baseConfig) > 0 { 85 | res.AddMeta(exptypes.ExporterImageBaseConfigKey, baseConfig) 86 | } 87 | } 88 | expPlatforms.Platforms[i] = exptypes.Platform{ 89 | ID: k, 90 | Platform: p, 91 | } 92 | return nil 93 | }) 94 | } 95 | if err := eg.Wait(); err != nil { 96 | return nil, err 97 | } 98 | return &ResultBuilder{ 99 | Result: res, 100 | expPlatforms: expPlatforms, 101 | }, nil 102 | } 103 | 104 | type ResultBuilder struct { 105 | *client.Result 106 | expPlatforms *exptypes.Platforms 107 | } 108 | 109 | func (rb *ResultBuilder) Finalize() (*client.Result, error) { 110 | dt, err := json.Marshal(rb.expPlatforms) 111 | if err != nil { 112 | return nil, err 113 | } 114 | rb.AddMeta(exptypes.ExporterPlatformsKey, dt) 115 | 116 | return rb.Result, nil 117 | } 118 | 119 | func (rb *ResultBuilder) EachPlatform(ctx context.Context, fn func(ctx context.Context, id string, p specs.Platform) error) error { 120 | eg, ctx := errgroup.WithContext(ctx) 121 | for _, p := range rb.expPlatforms.Platforms { 122 | p := p 123 | eg.Go(func() error { 124 | return fn(ctx, p.ID, p.Platform) 125 | }) 126 | } 127 | return eg.Wait() 128 | } 129 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerui/context.go: -------------------------------------------------------------------------------- 1 | package ggufpackerui 2 | 3 | import ( 4 | "archive/tar" 5 | "bytes" 6 | "context" 7 | "path/filepath" 8 | "regexp" 9 | "strconv" 10 | 11 | "github.com/moby/buildkit/client/llb" 12 | "github.com/moby/buildkit/frontend/gateway/client" 13 | gwpb "github.com/moby/buildkit/frontend/gateway/pb" 14 | "github.com/moby/buildkit/util/gitutil" 15 | "github.com/pkg/errors" 16 | ) 17 | 18 | const ( 19 | DefaultLocalNameContext = "context" 20 | DefaultGGUFPackerfileName = "GGUFPackerfile" 21 | DefaultGGUFPackerignoreName = ".ggufpackerignore" 22 | EmptyImageName = "scratch" 23 | ) 24 | 25 | const ( 26 | keyFilename = "filename" 27 | keyContextSubDir = "contextsubdir" 28 | keyNameContext = "contextkey" 29 | keyNameGGUFPackerfile = "ggufpackerfilekey" 30 | ) 31 | 32 | var httpPrefix = regexp.MustCompile(`^https?://`) 33 | 34 | type buildContext struct { 35 | context *llb.State // set if not local 36 | ggufpackerfile *llb.State // override remoteContext if set 37 | contextLocalName string 38 | ggufpackerfileLocalName string 39 | filename string 40 | forceLocalGGUFPackerfile bool 41 | } 42 | 43 | func (bc *Client) marshalOpts() []llb.ConstraintsOpt { 44 | return []llb.ConstraintsOpt{llb.WithCaps(bc.bopts.Caps)} 45 | } 46 | 47 | func (bc *Client) initContext(ctx context.Context) (*buildContext, error) { 48 | opts := bc.bopts.Opts 49 | gwcaps := bc.bopts.Caps 50 | 51 | localNameContext := DefaultLocalNameContext 52 | if v, ok := opts[keyNameContext]; ok { 53 | localNameContext = v 54 | } 55 | 56 | bctx := &buildContext{ 57 | contextLocalName: DefaultLocalNameContext, 58 | ggufpackerfileLocalName: DefaultLocalNameContext, 59 | filename: DefaultGGUFPackerfileName, 60 | } 61 | 62 | if v, ok := opts[keyFilename]; ok { 63 | bctx.filename = v 64 | } 65 | 66 | if v, ok := opts[keyNameGGUFPackerfile]; ok { 67 | bctx.forceLocalGGUFPackerfile = true 68 | bctx.ggufpackerfileLocalName = v 69 | } 70 | 71 | keepGit := false 72 | if v, err := strconv.ParseBool(opts[keyContextKeepGitDirArg]); err == nil { 73 | keepGit = v 74 | } 75 | if st, ok := DetectGitContext(opts[localNameContext], keepGit); ok { 76 | bctx.context = st 77 | bctx.ggufpackerfile = st 78 | } else if st, filename, ok := DetectHTTPContext(opts[localNameContext]); ok { 79 | def, err := st.Marshal(ctx, bc.marshalOpts()...) 80 | if err != nil { 81 | return nil, errors.Wrapf(err, "failed to marshal httpcontext") 82 | } 83 | res, err := bc.client.Solve(ctx, client.SolveRequest{ 84 | Definition: def.ToPB(), 85 | }) 86 | if err != nil { 87 | return nil, errors.Wrapf(err, "failed to resolve httpcontext") 88 | } 89 | 90 | ref, err := res.SingleRef() 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | dt, err := ref.ReadFile(ctx, client.ReadRequest{ 96 | Filename: filename, 97 | Range: &client.FileRange{ 98 | Length: 1024, 99 | }, 100 | }) 101 | if err != nil { 102 | return nil, errors.Wrapf(err, "failed to read downloaded context") 103 | } 104 | if isArchive(dt) { 105 | bc := llb.Scratch().File(llb.Copy(*st, filepath.Join("/", filename), "/", &llb.CopyInfo{ 106 | AttemptUnpack: true, 107 | })) 108 | bctx.context = &bc 109 | } else { 110 | bctx.filename = filename 111 | bctx.context = st 112 | } 113 | bctx.ggufpackerfile = bctx.context 114 | } else if (&gwcaps).Supports(gwpb.CapFrontendInputs) == nil { 115 | inputs, err := bc.client.Inputs(ctx) 116 | if err != nil { 117 | return nil, errors.Wrapf(err, "failed to get frontend inputs") 118 | } 119 | 120 | if !bctx.forceLocalGGUFPackerfile { 121 | inputGGUFPackerfile, ok := inputs[bctx.ggufpackerfileLocalName] 122 | if ok { 123 | bctx.ggufpackerfile = &inputGGUFPackerfile 124 | } 125 | } 126 | 127 | inputCtx, ok := inputs[DefaultLocalNameContext] 128 | if ok { 129 | bctx.context = &inputCtx 130 | } 131 | } 132 | 133 | if bctx.context != nil { 134 | if sub, ok := opts[keyContextSubDir]; ok { 135 | bctx.context = scopeToSubDir(bctx.context, sub) 136 | } 137 | } 138 | 139 | return bctx, nil 140 | } 141 | 142 | func DetectGitContext(ref string, keepGit bool) (*llb.State, bool) { 143 | g, err := gitutil.ParseGitRef(ref) 144 | if err != nil { 145 | return nil, false 146 | } 147 | commit := g.Commit 148 | if g.SubDir != "" { 149 | commit += ":" + g.SubDir 150 | } 151 | gitOpts := []llb.GitOption{WithInternalName("load git source " + ref)} 152 | if keepGit { 153 | gitOpts = append(gitOpts, llb.KeepGitDir()) 154 | } 155 | 156 | st := llb.Git(g.Remote, commit, gitOpts...) 157 | return &st, true 158 | } 159 | 160 | func DetectHTTPContext(ref string) (*llb.State, string, bool) { 161 | filename := "context" 162 | if httpPrefix.MatchString(ref) { 163 | st := llb.HTTP(ref, llb.Filename(filename), WithInternalName("load remote build context")) 164 | return &st, filename, true 165 | } 166 | return nil, "", false 167 | } 168 | 169 | func isArchive(header []byte) bool { 170 | for _, m := range [][]byte{ 171 | {0x42, 0x5A, 0x68}, // bzip2 172 | {0x1F, 0x8B, 0x08}, // gzip 173 | {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, // xz 174 | } { 175 | if len(header) < len(m) { 176 | continue 177 | } 178 | if bytes.Equal(m, header[:len(m)]) { 179 | return true 180 | } 181 | } 182 | 183 | r := tar.NewReader(bytes.NewBuffer(header)) 184 | _, err := r.Next() 185 | return err == nil 186 | } 187 | 188 | func scopeToSubDir(c *llb.State, dir string) *llb.State { 189 | bc := llb.Scratch().File(llb.Copy(*c, dir, "/", &llb.CopyInfo{ 190 | CopyDirContentsOnly: true, 191 | })) 192 | return &bc 193 | } 194 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerui/namedcontext.go: -------------------------------------------------------------------------------- 1 | package ggufpackerui 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "strings" 9 | "sync" 10 | 11 | "github.com/distribution/reference" 12 | "github.com/moby/buildkit/client/llb" 13 | "github.com/moby/buildkit/client/llb/sourceresolver" 14 | "github.com/moby/buildkit/exporter/containerimage/exptypes" 15 | "github.com/moby/buildkit/frontend/gateway/client" 16 | "github.com/moby/buildkit/solver/pb" 17 | "github.com/moby/buildkit/util/imageutil" 18 | "github.com/moby/patternmatcher/ignorefile" 19 | "github.com/pkg/errors" 20 | 21 | specs "github.com/gpustack/gguf-packer-go/buildkit/frontend/specs/v1" 22 | ) 23 | 24 | const ( 25 | contextPrefix = "context:" 26 | inputMetadataPrefix = "input-metadata:" 27 | maxContextRecursion = 10 28 | ) 29 | 30 | func (bc *Client) namedContext(ctx context.Context, name string, nameWithPlatform string, opt ContextOpt) (*llb.State, *specs.Image, error) { 31 | return bc.namedContextRecursive(ctx, name, nameWithPlatform, opt, 0) 32 | } 33 | 34 | func (bc *Client) namedContextRecursive(ctx context.Context, name string, nameWithPlatform string, opt ContextOpt, count int) (*llb.State, *specs.Image, error) { 35 | opts := bc.bopts.Opts 36 | contextKey := contextPrefix + nameWithPlatform 37 | v, ok := opts[contextKey] 38 | if !ok { 39 | return nil, nil, nil 40 | } 41 | 42 | if count > maxContextRecursion { 43 | return nil, nil, errors.New("context recursion limit exceeded; this may indicate a cycle in the provided source policies: " + v) 44 | } 45 | 46 | vv := strings.SplitN(v, ":", 2) 47 | if len(vv) != 2 { 48 | return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, nameWithPlatform) 49 | } 50 | 51 | // allow git@ without protocol for SSH URLs for backwards compatibility 52 | if strings.HasPrefix(vv[0], "git@") { 53 | vv[0] = "git" 54 | } 55 | switch vv[0] { 56 | case "docker-image": 57 | ref := strings.TrimPrefix(vv[1], "//") 58 | if ref == EmptyImageName { 59 | st := llb.Scratch() 60 | return &st, nil, nil 61 | } 62 | 63 | imgOpt := []llb.ImageOption{ 64 | llb.WithCustomName("[context " + nameWithPlatform + "] " + ref), 65 | } 66 | if opt.Platform != nil { 67 | imgOpt = append(imgOpt, llb.Platform(*opt.Platform)) 68 | } 69 | 70 | named, err := reference.ParseNormalizedNamed(ref) 71 | if err != nil { 72 | return nil, nil, err 73 | } 74 | 75 | named = reference.TagNameOnly(named) 76 | 77 | ref, dgst, data, err := bc.client.ResolveImageConfig(ctx, named.String(), sourceresolver.Opt{ 78 | LogName: fmt.Sprintf("[context %s] load metadata for %s", nameWithPlatform, ref), 79 | Platform: opt.Platform, 80 | ImageOpt: &sourceresolver.ResolveImageOpt{ 81 | ResolveMode: opt.ResolveMode, 82 | }, 83 | }) 84 | if err != nil { 85 | e := &imageutil.ResolveToNonImageError{} 86 | if errors.As(err, &e) { 87 | before, after, ok := strings.Cut(e.Updated, "://") 88 | if !ok { 89 | return nil, nil, errors.Errorf("could not parse ref: %s", e.Updated) 90 | } 91 | 92 | bc.bopts.Opts[contextKey] = before + ":" + after 93 | return bc.namedContextRecursive(ctx, name, nameWithPlatform, opt, count+1) 94 | } 95 | return nil, nil, err 96 | } 97 | 98 | var img specs.Image 99 | if err := json.Unmarshal(data, &img); err != nil { 100 | return nil, nil, err 101 | } 102 | img.Created = nil 103 | 104 | st := llb.Image(ref, imgOpt...) 105 | st, err = st.WithImageConfig(data) 106 | if err != nil { 107 | return nil, nil, err 108 | } 109 | if opt.CaptureDigest != nil { 110 | *opt.CaptureDigest = dgst 111 | } 112 | return &st, &img, nil 113 | case "git": 114 | st, ok := DetectGitContext(v, true) 115 | if !ok { 116 | return nil, nil, errors.Errorf("invalid git context %s", v) 117 | } 118 | return st, nil, nil 119 | case "http", "https": 120 | st, ok := DetectGitContext(v, true) 121 | if !ok { 122 | httpst := llb.HTTP(v, llb.WithCustomName("[context "+nameWithPlatform+"] "+v)) 123 | st = &httpst 124 | } 125 | return st, nil, nil 126 | case "oci-layout": 127 | refSpec := strings.TrimPrefix(vv[1], "//") 128 | ref, err := reference.Parse(refSpec) 129 | if err != nil { 130 | return nil, nil, errors.Wrapf(err, "could not parse oci-layout reference %q", refSpec) 131 | } 132 | named, ok := ref.(reference.Named) 133 | if !ok { 134 | return nil, nil, errors.Errorf("oci-layout reference %q has no name", ref.String()) 135 | } 136 | dgstd, ok := named.(reference.Digested) 137 | if !ok { 138 | return nil, nil, errors.Errorf("oci-layout reference %q has no digest", named.String()) 139 | } 140 | 141 | // for the dummy ref primarily used in log messages, we can use the 142 | // original name, since the store key may not be significant 143 | dummyRef, err := reference.ParseNormalizedNamed(name) 144 | if err != nil { 145 | return nil, nil, errors.Wrapf(err, "could not parse oci-layout reference %q", name) 146 | } 147 | dummyRef, err = reference.WithDigest(dummyRef, dgstd.Digest()) 148 | if err != nil { 149 | return nil, nil, errors.Wrapf(err, "could not wrap %q with digest", name) 150 | } 151 | 152 | _, dgst, data, err := bc.client.ResolveImageConfig(ctx, dummyRef.String(), sourceresolver.Opt{ 153 | LogName: fmt.Sprintf("[context %s] load metadata for %s", nameWithPlatform, dummyRef.String()), 154 | Platform: opt.Platform, 155 | OCILayoutOpt: &sourceresolver.ResolveOCILayoutOpt{ 156 | Store: sourceresolver.ResolveImageConfigOptStore{ 157 | SessionID: bc.bopts.SessionID, 158 | StoreID: named.Name(), 159 | }, 160 | }, 161 | }) 162 | if err != nil { 163 | return nil, nil, err 164 | } 165 | 166 | var img specs.Image 167 | if err := json.Unmarshal(data, &img); err != nil { 168 | return nil, nil, errors.Wrap(err, "could not parse oci-layout image config") 169 | } 170 | 171 | ociOpt := []llb.OCILayoutOption{ 172 | llb.WithCustomName("[context " + nameWithPlatform + "] OCI load from client"), 173 | llb.OCIStore(bc.bopts.SessionID, named.Name()), 174 | } 175 | if opt.Platform != nil { 176 | ociOpt = append(ociOpt, llb.Platform(*opt.Platform)) 177 | } 178 | st := llb.OCILayout( 179 | dummyRef.String(), 180 | ociOpt..., 181 | ) 182 | st, err = st.WithImageConfig(data) 183 | if err != nil { 184 | return nil, nil, err 185 | } 186 | if opt.CaptureDigest != nil { 187 | *opt.CaptureDigest = dgst 188 | } 189 | return &st, &img, nil 190 | case "local": 191 | st := llb.Local(vv[1], 192 | llb.SessionID(bc.bopts.SessionID), 193 | llb.FollowPaths([]string{DefaultGGUFPackerignoreName}), 194 | llb.SharedKeyHint("context:"+nameWithPlatform+"-"+DefaultGGUFPackerignoreName), 195 | llb.WithCustomName("[context "+nameWithPlatform+"] load "+DefaultGGUFPackerignoreName), 196 | llb.Differ(llb.DiffNone, false), 197 | ) 198 | def, err := st.Marshal(ctx) 199 | if err != nil { 200 | return nil, nil, err 201 | } 202 | res, err := bc.client.Solve(ctx, client.SolveRequest{ 203 | Evaluate: true, 204 | Definition: def.ToPB(), 205 | }) 206 | if err != nil { 207 | return nil, nil, err 208 | } 209 | ref, err := res.SingleRef() 210 | if err != nil { 211 | return nil, nil, err 212 | } 213 | 214 | var excludes []string 215 | dt, _ := ref.ReadFile(ctx, client.ReadRequest{ 216 | Filename: DefaultGGUFPackerignoreName, 217 | }) // error ignored 218 | 219 | if len(dt) != 0 { 220 | excludes, err = ignorefile.ReadAll(bytes.NewBuffer(dt)) 221 | if err != nil { 222 | return nil, nil, errors.Wrapf(err, "failed parsing %s", DefaultGGUFPackerignoreName) 223 | } 224 | } 225 | 226 | localOutput := &asyncLocalOutput{ 227 | name: vv[1], 228 | nameWithPlatform: nameWithPlatform, 229 | sessionID: bc.bopts.SessionID, 230 | excludes: excludes, 231 | extraOpts: opt.AsyncLocalOpts, 232 | } 233 | st = llb.NewState(localOutput) 234 | return &st, nil, nil 235 | case "input": 236 | inputs, err := bc.client.Inputs(ctx) 237 | if err != nil { 238 | return nil, nil, err 239 | } 240 | st, ok := inputs[vv[1]] 241 | if !ok { 242 | return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], nameWithPlatform) 243 | } 244 | md, ok := opts[inputMetadataPrefix+vv[1]] 245 | if ok { 246 | m := make(map[string][]byte) 247 | if err := json.Unmarshal([]byte(md), &m); err != nil { 248 | return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) 249 | } 250 | var img *specs.Image 251 | if dtic, ok := m[exptypes.ExporterImageConfigKey]; ok { 252 | st, err = st.WithImageConfig(dtic) 253 | if err != nil { 254 | return nil, nil, err 255 | } 256 | if err := json.Unmarshal(dtic, &img); err != nil { 257 | return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", nameWithPlatform) 258 | } 259 | } 260 | return &st, img, nil 261 | } 262 | return &st, nil, nil 263 | default: 264 | return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], nameWithPlatform) 265 | } 266 | } 267 | 268 | // asyncLocalOutput is an llb.Output that computes an llb.Local 269 | // on-demand instead of at the time of initialization. 270 | type asyncLocalOutput struct { 271 | llb.Output 272 | name string 273 | nameWithPlatform string 274 | sessionID string 275 | excludes []string 276 | extraOpts func() []llb.LocalOption 277 | once sync.Once 278 | } 279 | 280 | func (a *asyncLocalOutput) ToInput(ctx context.Context, constraints *llb.Constraints) (*pb.Input, error) { 281 | a.once.Do(a.do) 282 | return a.Output.ToInput(ctx, constraints) 283 | } 284 | 285 | func (a *asyncLocalOutput) Vertex(ctx context.Context, constraints *llb.Constraints) llb.Vertex { 286 | a.once.Do(a.do) 287 | return a.Output.Vertex(ctx, constraints) 288 | } 289 | 290 | func (a *asyncLocalOutput) do() { 291 | var extraOpts []llb.LocalOption 292 | if a.extraOpts != nil { 293 | extraOpts = a.extraOpts() 294 | } 295 | opts := append([]llb.LocalOption{ 296 | llb.WithCustomName("[context " + a.nameWithPlatform + "] load from client"), 297 | llb.SessionID(a.sessionID), 298 | llb.SharedKeyHint("context:" + a.nameWithPlatform), 299 | llb.ExcludePatterns(a.excludes), 300 | }, extraOpts...) 301 | 302 | st := llb.Local(a.name, opts...) 303 | a.Output = st.Output() 304 | } 305 | -------------------------------------------------------------------------------- /buildkit/frontend/ggufpackerui/requests.go: -------------------------------------------------------------------------------- 1 | package ggufpackerui 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | 8 | "github.com/moby/buildkit/frontend/gateway/client" 9 | "github.com/moby/buildkit/frontend/subrequests" 10 | "github.com/moby/buildkit/frontend/subrequests/lint" 11 | "github.com/moby/buildkit/frontend/subrequests/outline" 12 | "github.com/moby/buildkit/frontend/subrequests/targets" 13 | "github.com/moby/buildkit/solver/errdefs" 14 | ) 15 | 16 | const ( 17 | keyRequestID = "requestid" 18 | ) 19 | 20 | type RequestHandler struct { 21 | Outline func(context.Context) (*outline.Outline, error) 22 | ListTargets func(context.Context) (*targets.List, error) 23 | Lint func(context.Context) (*lint.LintResults, error) 24 | AllowOther bool 25 | } 26 | 27 | func (bc *Client) HandleSubrequest(ctx context.Context, h RequestHandler) (*client.Result, bool, error) { 28 | req, ok := bc.bopts.Opts[keyRequestID] 29 | if !ok { 30 | return nil, false, nil 31 | } 32 | switch req { 33 | case subrequests.RequestSubrequestsDescribe: 34 | res, err := describe(h) 35 | return res, true, err 36 | case outline.SubrequestsOutlineDefinition.Name: 37 | if f := h.Outline; f != nil { 38 | o, err := f(ctx) 39 | if err != nil { 40 | return nil, false, err 41 | } 42 | if o == nil { 43 | return nil, true, nil 44 | } 45 | res, err := o.ToResult() 46 | return res, true, err 47 | } 48 | case targets.SubrequestsTargetsDefinition.Name: 49 | if f := h.ListTargets; f != nil { 50 | targets, err := f(ctx) 51 | if err != nil { 52 | return nil, false, err 53 | } 54 | if targets == nil { 55 | return nil, true, nil 56 | } 57 | res, err := targets.ToResult() 58 | return res, true, err 59 | } 60 | case lint.SubrequestLintDefinition.Name: 61 | if f := h.Lint; f != nil { 62 | warnings, err := f(ctx) 63 | if err != nil { 64 | return nil, false, err 65 | } 66 | if warnings == nil { 67 | return nil, true, nil 68 | } 69 | res, err := warnings.ToResult() 70 | return res, true, err 71 | } 72 | } 73 | if h.AllowOther { 74 | return nil, false, nil 75 | } 76 | return nil, false, errdefs.NewUnsupportedSubrequestError(req) 77 | } 78 | 79 | func describe(h RequestHandler) (*client.Result, error) { 80 | all := []subrequests.Request{} 81 | if h.Outline != nil { 82 | all = append(all, outline.SubrequestsOutlineDefinition) 83 | } 84 | if h.ListTargets != nil { 85 | all = append(all, targets.SubrequestsTargetsDefinition) 86 | } 87 | all = append(all, subrequests.SubrequestsDescribeDefinition) 88 | dt, err := json.MarshalIndent(all, "", " ") 89 | if err != nil { 90 | return nil, err 91 | } 92 | 93 | b := bytes.NewBuffer(nil) 94 | if err := subrequests.PrintDescribe(dt, b); err != nil { 95 | return nil, err 96 | } 97 | 98 | res := client.NewResult() 99 | res.Metadata = map[string][]byte{ 100 | "result.json": dt, 101 | "result.txt": b.Bytes(), 102 | "version": []byte(subrequests.SubrequestsDescribeDefinition.Version), 103 | } 104 | return res, nil 105 | } 106 | -------------------------------------------------------------------------------- /buildkit/frontend/specs/v1/image.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "time" 5 | 6 | ggufparser "github.com/gpustack/gguf-parser-go" 7 | ocispec "github.com/opencontainers/image-spec/specs-go/v1" 8 | ) 9 | 10 | type ( 11 | Image struct { 12 | // Created is the combined date and time at which the image was created, formatted as defined by RFC 3339, section 5.6. 13 | Created *time.Time `json:"created,omitempty"` 14 | 15 | // Author defines the name and/or email address of the person or entity which created and is responsible for maintaining the image. 16 | Author string `json:"author,omitempty"` 17 | 18 | // Platform describes the platform which the image in the manifest runs on. 19 | Platform 20 | 21 | // Config defines the execution parameters which should be used as a base when running a container using the image. 22 | Config ImageConfig `json:"config,omitempty"` 23 | 24 | // RootFS references the layer content addresses used by the image. 25 | RootFS RootFS `json:"rootfs"` 26 | 27 | // History describes the history of each layer. 28 | History []History `json:"history,omitempty"` 29 | } 30 | 31 | Platform = ocispec.Platform 32 | 33 | ImageConfig struct { 34 | // Size represents the summarized size of all GGUFFiles. 35 | Size ggufparser.GGUFBytesScalar `json:"Size,omitempty"` 36 | 37 | // Model represents the main model of the image. 38 | Model *GGUFFile `json:"Model,omitempty"` 39 | 40 | // Drafter represents the drafter of the image. 41 | Drafter *GGUFFile `json:"Drafter,omitempty"` 42 | 43 | // Projector represents the projector of the image. 44 | Projector *GGUFFile `json:"Projector,omitempty"` 45 | 46 | // Adapters represents the adapters of the image. 47 | Adapters []*GGUFFile `json:"Adapters,omitempty"` 48 | 49 | // Cmd defines the arguments to launch. 50 | Cmd []string `json:"Cmd,omitempty"` 51 | 52 | // Labels contains arbitrary metadata for the image. 53 | Labels map[string]string `json:"Labels,omitempty"` 54 | } 55 | 56 | RootFS = ocispec.RootFS 57 | 58 | History = ocispec.History 59 | 60 | GGUFFile struct { 61 | ggufparser.GGUFFile `json:"GGUF"` 62 | 63 | // Architecture represents what architecture the model implements. 64 | Architecture string `json:"Architecture,omitempty"` 65 | 66 | // Parameters represents the parameters of the model. 67 | Parameters ggufparser.GGUFParametersScalar `json:"Parameters"` 68 | 69 | // BitsPerWeight represents the bits per weight of the model. 70 | BitsPerWeight ggufparser.GGUFBitsPerWeightScalar `json:"BitsPerWeight"` 71 | 72 | // FileType represents the file type of the model. 73 | FileType ggufparser.GGUFFileType `json:"FileType,omitempty"` 74 | 75 | // CmdParameterValue indicates the parameter value of the Cmd. 76 | CmdParameterValue string `json:"CmdParameterValue,omitempty"` 77 | 78 | // CmdParameterIndex indicates the index of the Cmd. 79 | CmdParameterIndex int `json:"CmdParameterIndex,omitempty"` 80 | } 81 | ) 82 | -------------------------------------------------------------------------------- /cmd/gguf-packer/README.md: -------------------------------------------------------------------------------- 1 | # GGUF Packer 2 | 3 | Deliver LLMs of [GGUF](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md) format via Dockerfile, 4 | see [https://github.com/gpustack/gguf-packer-go](https://github.com/gpustack/gguf-packer-go). 5 | 6 | ## Usage 7 | 8 | ```shell 9 | $ gguf-packer --help 10 | Pack the GGUF format model. 11 | 12 | Usage: 13 | gguf-packer [command] 14 | 15 | Examples: 16 | # Serve as BuildKit frontend 17 | gguf-packer llb-frontend 18 | 19 | # Dump the BuildKit LLB of the current directory 20 | gguf-packer llb-dump 21 | 22 | # Pull the model from the registry 23 | gguf-packer pull gpustack/qwen2:0.5b-instruct 24 | 25 | # Inspect the model 26 | gguf-packer inspect gpustack/qwen2:0.5b-instruct 27 | 28 | # Estimate the model memory usage 29 | gguf-packer estimate gpustack/qwen2:0.5b-instruct 30 | 31 | # List all local models 32 | gguf-packer list 33 | 34 | # Remove a local model 35 | gguf-packer remove gpustack/qwen2:0.5b-instruct 36 | 37 | # Run a model by container container: ghcr.io/ggerganov/llama.cpp:server 38 | gguf-packer run gpustack/qwen2:0.5b-instruct 39 | 40 | Available Commands: 41 | estimate Estimate the model memory usage. 42 | help Help about any command 43 | inspect Get the low-level information of a model. 44 | list List all local models. 45 | llb-dump Dump the BuildKit LLB of the GGUFPackerfile. 46 | llb-frontend Serve as BuildKit frontend. 47 | pull Download a model from a registry. 48 | remove Remove one or more local models. 49 | run Run a model by specific process, like container image or executable binary. 50 | 51 | Flags: 52 | -h, --help help for gguf-packer 53 | -v, --version version for gguf-packer 54 | 55 | Use "gguf-packer [command] --help" for more information about a command. 56 | 57 | ``` 58 | 59 | ## License 60 | 61 | MIT 62 | -------------------------------------------------------------------------------- /cmd/gguf-packer/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gpustack/gguf-packer-go/cmd/gguf-packer 2 | 3 | go 1.22.0 4 | 5 | toolchain go1.22.3 6 | 7 | replace github.com/gpustack/gguf-packer-go => ../../ 8 | 9 | require ( 10 | github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240809155957-ac94a3401898 11 | github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 12 | github.com/containerd/containerd/v2 v2.0.0-rc.3 13 | github.com/distribution/reference v0.6.0 14 | github.com/dustin/go-humanize v1.0.1 15 | github.com/google/go-containerregistry v0.20.2 16 | github.com/gpustack/gguf-packer-go v0.0.0-00010101000000-000000000000 17 | github.com/gpustack/gguf-parser-go v0.12.0 18 | github.com/jedib0t/go-pretty/v6 v6.5.9 19 | github.com/moby/buildkit v0.15.2 20 | github.com/opencontainers/go-digest v1.0.0 21 | github.com/schollz/progressbar/v3 v3.14.6 22 | github.com/spf13/cobra v1.8.1 23 | ) 24 | 25 | require ( 26 | cloud.google.com/go/compute/metadata v0.5.0 // indirect 27 | github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect 28 | github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect 29 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect 30 | github.com/Azure/go-autorest/autorest v0.11.29 // indirect 31 | github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect 32 | github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 // indirect 33 | github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect 34 | github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect 35 | github.com/Azure/go-autorest/logger v0.2.1 // indirect 36 | github.com/Azure/go-autorest/tracing v0.6.0 // indirect 37 | github.com/Microsoft/go-winio v0.6.2 // indirect 38 | github.com/Microsoft/hcsshim v0.12.5 // indirect 39 | github.com/agext/levenshtein v1.2.3 // indirect 40 | github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect 41 | github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect 42 | github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect 43 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect 44 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect 45 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect 46 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect 47 | github.com/aws/aws-sdk-go-v2/service/ecr v1.32.0 // indirect 48 | github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.3 // indirect 49 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect 50 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect 51 | github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect 52 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect 53 | github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect 54 | github.com/aws/smithy-go v1.20.3 // indirect 55 | github.com/containerd/cgroups/v3 v3.0.3 // indirect 56 | github.com/containerd/containerd v1.7.20 // indirect 57 | github.com/containerd/containerd/api v1.8.0-rc.2 // indirect 58 | github.com/containerd/continuity v0.4.3 // indirect 59 | github.com/containerd/errdefs v0.1.0 // indirect 60 | github.com/containerd/fifo v1.1.0 // indirect 61 | github.com/containerd/log v0.1.0 // indirect 62 | github.com/containerd/nydus-snapshotter v0.14.0 // indirect 63 | github.com/containerd/platforms v0.2.1 // indirect 64 | github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect 65 | github.com/containerd/ttrpc v1.2.5 // indirect 66 | github.com/containerd/typeurl/v2 v2.2.0 // indirect 67 | github.com/dimchansky/utfbom v1.1.1 // indirect 68 | github.com/docker/cli v27.1.1+incompatible // indirect 69 | github.com/docker/distribution v2.8.3+incompatible // indirect 70 | github.com/docker/docker v27.1.1+incompatible // indirect 71 | github.com/docker/docker-credential-helpers v0.8.2 // indirect 72 | github.com/docker/go-units v0.5.0 // indirect 73 | github.com/felixge/httpsnoop v1.0.4 // indirect 74 | github.com/go-logr/logr v1.4.2 // indirect 75 | github.com/go-logr/stdr v1.2.2 // indirect 76 | github.com/gofrs/flock v0.12.1 // indirect 77 | github.com/gogo/googleapis v1.4.1 // indirect 78 | github.com/gogo/protobuf v1.3.2 // indirect 79 | github.com/golang-jwt/jwt/v4 v4.5.0 // indirect 80 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 81 | github.com/golang/protobuf v1.5.4 // indirect 82 | github.com/google/go-cmp v0.6.0 // indirect 83 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect 84 | github.com/google/uuid v1.6.0 // indirect 85 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 // indirect 86 | github.com/hashicorp/errwrap v1.1.0 // indirect 87 | github.com/hashicorp/go-multierror v1.1.1 // indirect 88 | github.com/henvic/httpretty v0.1.3 // indirect 89 | github.com/in-toto/in-toto-golang v0.9.0 // indirect 90 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 91 | github.com/jmespath/go-jmespath v0.4.0 // indirect 92 | github.com/json-iterator/go v1.1.12 // indirect 93 | github.com/klauspost/compress v1.17.9 // indirect 94 | github.com/mattn/go-runewidth v0.0.16 // indirect 95 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect 96 | github.com/mitchellh/go-homedir v1.1.0 // indirect 97 | github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect 98 | github.com/moby/docker-image-spec v1.3.1 // indirect 99 | github.com/moby/locker v1.0.1 // indirect 100 | github.com/moby/patternmatcher v0.6.0 // indirect 101 | github.com/moby/sys/mountinfo v0.7.2 // indirect 102 | github.com/moby/sys/sequential v0.6.0 // indirect 103 | github.com/moby/sys/signal v0.7.1 // indirect 104 | github.com/moby/sys/user v0.3.0 // indirect 105 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 106 | github.com/modern-go/reflect2 v1.0.2 // indirect 107 | github.com/opencontainers/image-spec v1.1.0 // indirect 108 | github.com/pkg/errors v0.9.1 // indirect 109 | github.com/rivo/uniseg v0.4.7 // indirect 110 | github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect 111 | github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect 112 | github.com/shibumi/go-pathspec v1.3.0 // indirect 113 | github.com/sirupsen/logrus v1.9.3 // indirect 114 | github.com/smallnest/ringbuffer v0.0.0-20240809045605-2fc0b613bd6b // indirect 115 | github.com/spf13/pflag v1.0.5 // indirect 116 | github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c // indirect 117 | github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 // indirect 118 | github.com/vbatts/tar-split v0.11.5 // indirect 119 | go.opencensus.io v0.24.0 // indirect 120 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect 121 | go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.53.0 // indirect 122 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect 123 | go.opentelemetry.io/otel v1.28.0 // indirect 124 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect 125 | go.opentelemetry.io/otel/metric v1.28.0 // indirect 126 | go.opentelemetry.io/otel/sdk v1.28.0 // indirect 127 | go.opentelemetry.io/otel/trace v1.28.0 // indirect 128 | go.opentelemetry.io/proto/otlp v1.3.1 // indirect 129 | golang.org/x/crypto v0.26.0 // indirect 130 | golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect 131 | golang.org/x/mod v0.20.0 // indirect 132 | golang.org/x/net v0.28.0 // indirect 133 | golang.org/x/oauth2 v0.22.0 // indirect 134 | golang.org/x/sync v0.8.0 // indirect 135 | golang.org/x/sys v0.25.0 // indirect 136 | golang.org/x/term v0.23.0 // indirect 137 | golang.org/x/text v0.17.0 // indirect 138 | golang.org/x/tools v0.24.0 // indirect 139 | google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 // indirect 140 | google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 // indirect 141 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 // indirect 142 | google.golang.org/grpc v1.65.0 // indirect 143 | google.golang.org/protobuf v1.34.2 // indirect 144 | ) 145 | -------------------------------------------------------------------------------- /cmd/gguf-packer/inspect.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "os" 8 | 9 | "github.com/google/go-containerregistry/pkg/crane" 10 | "github.com/google/go-containerregistry/pkg/name" 11 | "github.com/google/go-containerregistry/pkg/v1/remote" 12 | specs "github.com/gpustack/gguf-packer-go/buildkit/frontend/specs/v1" 13 | "github.com/gpustack/gguf-packer-go/util/osx" 14 | "github.com/spf13/cobra" 15 | ) 16 | 17 | func inspect(app string) *cobra.Command { 18 | var ( 19 | insecure bool 20 | force bool 21 | ) 22 | 23 | c := &cobra.Command{ 24 | Use: "inspect MODEL", 25 | Short: "Get the low-level information of a model.", 26 | Example: sprintf(` # Inspect a model 27 | %s inspect gpustack/qwen2:0.5b-instruct 28 | 29 | # Force inspect a model from remote 30 | %[1]s inspect gpustack/qwen2:0.5b-instruct --force`, app), 31 | Args: cobra.ExactArgs(1), 32 | RunE: func(c *cobra.Command, args []string) error { 33 | model := args[0] 34 | 35 | var cos crane.Options 36 | { 37 | co := []crane.Option{ 38 | getAuthnKeychainOption(), 39 | } 40 | if insecure { 41 | co = append(co, crane.Insecure) 42 | } 43 | cos = crane.GetOptions(co...) 44 | } 45 | 46 | rf, err := name.NewTag(model, cos.Name...) 47 | if err != nil { 48 | return fmt.Errorf("parsing model reference %q: %w", model, err) 49 | } 50 | 51 | cf, err := retrieveConfigByOCIReference(force, rf, cos.Remote...) 52 | if err != nil { 53 | return err 54 | } 55 | cf.History = nil // Remove history. 56 | jprint(c.OutOrStdout(), cf) 57 | return nil 58 | }, 59 | } 60 | c.Flags().BoolVar(&insecure, "insecure", insecure, "Allow model references to be fetched without TLS.") 61 | c.Flags().BoolVar(&force, "force", force, "Always inspect the model from the registry.") 62 | return c 63 | } 64 | 65 | func retrieveConfigByOCIReference(force bool, ref name.Reference, opts ...remote.Option) (cf specs.Image, err error) { 66 | // Read from local. 67 | if !force { 68 | mdp := getModelMetadataStorePath(ref) 69 | if osx.ExistsLink(mdp) { 70 | return retrieveConfigByPath(mdp) 71 | } 72 | } 73 | 74 | // Otherwise, read from remote. 75 | rd, err := remote.Get(ref, opts...) 76 | if err != nil { 77 | return cf, fmt.Errorf("getting model remote %q: %w", ref.Name(), err) 78 | } 79 | img, err := retrieveOCIImage(rd) 80 | if err != nil { 81 | return cf, err 82 | } 83 | cf, _, err = retrieveConfigByOCIImage(img) 84 | return cf, err 85 | } 86 | 87 | func retrieveConfigByPath(cfp string) (cf specs.Image, err error) { 88 | cfBs, err := os.ReadFile(cfp) 89 | if err != nil { 90 | return cf, fmt.Errorf("reading model config: %w", err) 91 | } 92 | if err = json.Unmarshal(cfBs, &cf); err != nil { 93 | return cf, fmt.Errorf("unmarshalling model config: %w", err) 94 | } 95 | if !isConfigAvailable(&cf) { 96 | return cf, errors.New("unavailable model config") 97 | } 98 | return cf, nil 99 | } 100 | -------------------------------------------------------------------------------- /cmd/gguf-packer/list.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strings" 7 | 8 | "github.com/dustin/go-humanize" 9 | "github.com/gpustack/gguf-packer-go/util/mapx" 10 | "github.com/gpustack/gguf-packer-go/util/osx" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | func list(app string) *cobra.Command { 15 | var ( 16 | fullID bool 17 | ) 18 | c := &cobra.Command{ 19 | Use: "list", 20 | Short: "List all local models.", 21 | Example: sprintf(` # List all local models 22 | %s list`, app), 23 | Args: cobra.ExactArgs(0), 24 | RunE: func(c *cobra.Command, args []string) error { 25 | var ( 26 | hds = [][]any{ 27 | { 28 | "Name", 29 | "Tag", 30 | "ID", 31 | "Arch", 32 | "Params", 33 | "Bpw", 34 | "Type", 35 | "Usage", 36 | "Created", 37 | "Size", 38 | }, 39 | } 40 | bds [][]any 41 | ) 42 | 43 | msdp := getModelsMetadataStorePath() 44 | if osx.ExistsDir(msdp) { 45 | _ = filepath.Walk(msdp, func(mdp string, info os.FileInfo, err error) error { 46 | if err != nil { 47 | return err 48 | } 49 | if info.IsDir() { 50 | if strings.HasPrefix(filepath.Base(mdp), ".") { 51 | return filepath.SkipDir 52 | } 53 | return nil 54 | } 55 | 56 | cfp, err := os.Readlink(mdp) 57 | if err != nil { 58 | // Ignore non-symbolic link. 59 | return nil 60 | } 61 | 62 | img, err := retrieveConfigByPath(cfp) 63 | if err != nil { 64 | // Ignore invalid model metadata. 65 | return nil 66 | } 67 | 68 | mname := strings.TrimPrefix(filepath.Dir(mdp), msdp+string(filepath.Separator)) 69 | mtag := filepath.Base(mdp) 70 | mid := filepath.Base(cfp) 71 | arch := img.Config.Model.Architecture 72 | params := img.Config.Model.Parameters 73 | bpw := img.Config.Model.BitsPerWeight 74 | fileType := img.Config.Model.FileType 75 | usage := mapx.Value(img.Config.Labels, "gguf.model.usage", "unknown") 76 | created := img.Created 77 | size := img.Config.Size 78 | 79 | bds = append(bds, []any{ 80 | sprintf(tenary(strings.HasPrefix(mname, dockerRegPrefix), mname[16:], mname)), 81 | sprintf(tenary(strings.HasPrefix(mtag, oldPrefix), "", mtag)), 82 | sprintf(tenary(fullID, mid, mid[:12])), 83 | sprintf(arch), 84 | sprintf(params), 85 | sprintf(bpw), 86 | sprintf(fileType), 87 | sprintf(usage), 88 | sprintf(tenary(created != nil, humanize.Time(*created), "unknown")), 89 | sprintf(size), 90 | }) 91 | return nil 92 | }) 93 | } 94 | 95 | tfprint(c.OutOrStdout(), false, hds, bds) 96 | return nil 97 | }, 98 | } 99 | c.Flags().BoolVar(&fullID, "full-id", false, "Display full model ID.") 100 | return c 101 | } 102 | -------------------------------------------------------------------------------- /cmd/gguf-packer/llb_dump.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "net/url" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/distribution/reference" 13 | ggufpacker "github.com/gpustack/gguf-packer-go" 14 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerfile/parser" 15 | "github.com/gpustack/gguf-packer-go/buildkit/frontend/ggufpackerui" 16 | "github.com/gpustack/gguf-packer-go/util/osx" 17 | "github.com/moby/buildkit/client/llb" 18 | "github.com/moby/buildkit/frontend/dockerui" 19 | "github.com/moby/buildkit/solver/pb" 20 | "github.com/opencontainers/go-digest" 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | func llbDump(app string) *cobra.Command { 25 | var ( 26 | browser bool 27 | json bool 28 | protobuf bool 29 | ) 30 | 31 | c := &cobra.Command{ 32 | Use: "llb-dump [PATH]", 33 | Short: "Dump the BuildKit LLB of the GGUFPackerfile.", 34 | Example: sprintf(` # Dump the BuildKit LLB of the current directory 35 | %s llb-dump 36 | 37 | # Dump the BuildKit LLB of the GGUFPackerfile 38 | %[1]s llb-dump /path/to/file 39 | 40 | # Dump the BuildKit LLB of a specific directory 41 | %[1]s llb-dump /path/to/dir 42 | 43 | # Dump the BuildKit LLB of the current directory as json format 44 | %[1]s llb-dump --json 45 | 46 | # Dump the BuildKit LLB of the current directory as protobuf format 47 | %[1]s llb-dump --protobuf | buildctl debug dump-llb`, app), 48 | Args: cobra.MaximumNArgs(1), 49 | RunE: func(c *cobra.Command, args []string) error { 50 | // Inspired by https://github.com/moby/buildkit/blob/bc92b63b98aa0968614240082997483f6bf68cbe/cmd/buildctl/debug/dumpllb.go#L30. 51 | var path string 52 | { 53 | if len(args) == 0 { 54 | pwd, err := os.Getwd() 55 | if err != nil { 56 | return fmt.Errorf("get working directory: %w", err) 57 | } 58 | path = pwd 59 | } else { 60 | path = osx.InlineTilde(args[0]) 61 | } 62 | } 63 | switch { 64 | case osx.ExistsDir(path): 65 | f := filepath.Join(path, ggufpackerui.DefaultGGUFPackerfileName) 66 | if !osx.ExistsFile(f) { 67 | f = filepath.Join(path, dockerui.DefaultDockerfileName) 68 | if !osx.ExistsFile(f) { 69 | return fmt.Errorf("cannot find target file under %s", path) 70 | } 71 | } 72 | path = f 73 | case !osx.ExistsFile(path): 74 | return errors.New("cannot find target file") 75 | } 76 | 77 | var st *llb.State 78 | { 79 | bs, err := os.ReadFile(path) 80 | if err != nil { 81 | return fmt.Errorf("read %s: %w", path, err) 82 | } 83 | if filepath.Base(path) == dockerui.DefaultDockerfileName { 84 | ref, _, _, ok := parser.DetectSyntax(bs) 85 | if !ok { 86 | return errors.New("cannot detect syntax") 87 | } 88 | nd, err := reference.ParseNormalizedNamed(ref) 89 | if err != nil { 90 | return fmt.Errorf("parse reference from detect syntax: %w", err) 91 | } 92 | if !strings.HasSuffix(nd.String(), "/gguf-packer") { 93 | return errors.New("invalid syntax") 94 | } 95 | } 96 | st, err = ggufpacker.ToLLB(c.Context(), bs) 97 | if err != nil { 98 | return fmt.Errorf("parse GGUFPackerfile to LLB: %w", err) 99 | } 100 | } 101 | 102 | def, err := st.Marshal(c.Context()) 103 | if err != nil { 104 | return fmt.Errorf("marshal LLB: %w", err) 105 | } 106 | 107 | w := c.OutOrStdout() 108 | if protobuf { 109 | return llb.WriteTo(def, w) 110 | } 111 | 112 | ops := make([]struct { 113 | Op pb.Op `json:"op"` 114 | Digest digest.Digest `json:"digest"` 115 | OpMetadata pb.OpMetadata `json:"opMetadata"` 116 | }, len(def.Def)) 117 | for i := range def.Def { 118 | if err = (&ops[i].Op).Unmarshal(def.Def[i]); err != nil { 119 | return fmt.Errorf("unmarshal op: %w", err) 120 | } 121 | ops[i].Digest = digest.FromBytes(def.Def[i]) 122 | ops[i].OpMetadata = def.Metadata[ops[i].Digest] 123 | } 124 | 125 | if json { 126 | jprint(w, ops) 127 | return nil 128 | } 129 | 130 | sb := &strings.Builder{} 131 | if browser { 132 | w = io.MultiWriter(w, sb) 133 | } 134 | fprintf(w, "digraph {\n") 135 | for _, op := range ops { 136 | name, shape := dotAttr(op.Digest, op.Op) 137 | fprintf(w, " %q [label=%q shape=%q];\n", op.Digest, name, shape) 138 | } 139 | for _, op := range ops { 140 | for i, inp := range op.Op.Inputs { 141 | label := "" 142 | if eo, ok := op.Op.Op.(*pb.Op_Exec); ok { 143 | for _, m := range eo.Exec.Mounts { 144 | if int(m.Input) == i && m.Dest != "/" { 145 | label = m.Dest 146 | } 147 | } 148 | } 149 | fprintf(w, " %q -> %q [label=%q];\n", inp.Digest, op.Digest, label) 150 | } 151 | } 152 | fprintf(w, "}\n") 153 | if !browser { 154 | return nil 155 | } 156 | err = osx.OpenBrowser("https://dreampuf.github.io/GraphvizOnline/#" + url.PathEscape(sb.String())) 157 | if err == nil { 158 | fprint(w, "\n!!!View the graph at your default browser!!!\n") 159 | } 160 | return nil 161 | }, 162 | } 163 | c.Flags().BoolVar(&browser, "browser", browser, "Output as dot format and try to display it on the default browser.") 164 | c.Flags().BoolVar(&json, "json", json, "Output as json format.") 165 | c.Flags().BoolVar(&protobuf, "protobuf", protobuf, "Output as protobuf format.") 166 | return c 167 | } 168 | 169 | func dotAttr(dgst digest.Digest, op pb.Op) (string, string) { 170 | switch op := op.Op.(type) { 171 | case *pb.Op_Source: 172 | return op.Source.Identifier, "ellipse" 173 | case *pb.Op_Exec: 174 | return strings.Join(op.Exec.Meta.Args, " "), "box" 175 | case *pb.Op_Build: 176 | return "build", "box3d" 177 | case *pb.Op_Merge: 178 | return "merge", "invtriangle" 179 | case *pb.Op_Diff: 180 | return "diff", "doublecircle" 181 | case *pb.Op_File: 182 | var names []string 183 | for _, action := range op.File.Actions { 184 | var name string 185 | switch act := action.Action.(type) { 186 | case *pb.FileAction_Copy: 187 | name = fmt.Sprintf("copy{src=%s, dest=%s}", act.Copy.Src, act.Copy.Dest) 188 | case *pb.FileAction_Mkfile: 189 | name = fmt.Sprintf("mkfile{path=%s}", act.Mkfile.Path) 190 | case *pb.FileAction_Mkdir: 191 | name = fmt.Sprintf("mkdir{path=%s}", act.Mkdir.Path) 192 | case *pb.FileAction_Rm: 193 | name = fmt.Sprintf("rm{path=%s}", act.Rm.Path) 194 | } 195 | names = append(names, name) 196 | } 197 | return strings.Join(names, ","), "note" 198 | default: 199 | return dgst.String(), "plaintext" 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /cmd/gguf-packer/llb_frontend.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | ggufpacker "github.com/gpustack/gguf-packer-go" 5 | "github.com/moby/buildkit/frontend/gateway/grpcclient" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | func llbFrontend(app string) *cobra.Command { 10 | c := &cobra.Command{ 11 | Use: "llb-frontend", 12 | Short: "Serve as BuildKit frontend.", 13 | Example: sprintf(` # Serve as BuildKit frontend 14 | %s llb-frontend`, app), 15 | Args: cobra.ExactArgs(0), 16 | RunE: func(c *cobra.Command, args []string) error { 17 | return grpcclient.RunFromEnvironment(c.Context(), ggufpacker.Build) 18 | }, 19 | } 20 | return c 21 | } 22 | -------------------------------------------------------------------------------- /cmd/gguf-packer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "log/slog" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/gpustack/gguf-packer-go/util/anyx" 13 | "github.com/gpustack/gguf-packer-go/util/osx" 14 | "github.com/gpustack/gguf-packer-go/util/signalx" 15 | "github.com/jedib0t/go-pretty/v6/table" 16 | "github.com/jedib0t/go-pretty/v6/text" 17 | "github.com/spf13/cobra" 18 | ) 19 | 20 | var ( 21 | Version = "v0.0.0" 22 | 23 | storePath string 24 | ) 25 | 26 | func main() { 27 | var ( 28 | stdin = os.Stdin 29 | stdout = os.Stdout 30 | stderr = os.Stderr 31 | ) 32 | storePath = osx.ExpandEnv("GGUF_PACKER_STORE_PATH") 33 | if storePath == "" { 34 | hd, err := os.UserHomeDir() 35 | if err != nil { 36 | _, _ = fmt.Fprintf(stderr, "getting home directory: %v\n", err) 37 | os.Exit(1) 38 | } 39 | storePath = filepath.Join(hd, "gguf-packer") 40 | } 41 | if err := os.MkdirAll(storePath, 0755); err != nil { 42 | _, _ = fmt.Fprintf(stderr, "creating store directory: %v\n", err) 43 | os.Exit(1) 44 | } 45 | slog.SetDefault(slog.New(slog.NewTextHandler(stderr, nil))) 46 | 47 | app := filepath.Base(os.Args[0]) 48 | root := &cobra.Command{ 49 | Version: Version, 50 | Use: app, 51 | Short: "Pack the GGUF format model.", 52 | CompletionOptions: cobra.CompletionOptions{ 53 | DisableDefaultCmd: true, 54 | }, 55 | Example: sprintf(` # Serve as BuildKit frontend 56 | %s llb-frontend 57 | 58 | # Dump the BuildKit LLB of the current directory 59 | %[1]s llb-dump 60 | 61 | # Pull the model from the registry 62 | %[1]s pull gpustack/qwen2:0.5b-instruct 63 | 64 | # Inspect the model 65 | %[1]s inspect gpustack/qwen2:0.5b-instruct 66 | 67 | # Estimate the model memory usage 68 | %[1]s estimate gpustack/qwen2:0.5b-instruct 69 | 70 | # List all local models 71 | %[1]s list 72 | 73 | # Remove a local model 74 | %[1]s remove gpustack/qwen2:0.5b-instruct 75 | 76 | # Run a model by container container: ghcr.io/ggerganov/llama.cpp:server 77 | %[1]s run gpustack/qwen2:0.5b-instruct`, app), 78 | } 79 | for _, cmdCreate := range []func(string) *cobra.Command{ 80 | llbFrontend, llbDump, inspect, pull, estimate, list, remove, run, 81 | } { 82 | cmd := cmdCreate(app) 83 | root.AddCommand(cmd) 84 | } 85 | root.SetIn(stdin) 86 | root.SetOut(stdout) 87 | root.SetErr(stderr) 88 | 89 | if err := root.ExecuteContext(signalx.Handler()); err != nil { 90 | _, _ = fmt.Fprintf(stderr, "%v\n", err) 91 | os.Exit(1) 92 | } 93 | } 94 | 95 | func getModelsStorePath() string { 96 | return filepath.Join(storePath, "models") 97 | } 98 | 99 | func getModelsMetadataStorePath() string { 100 | return filepath.Join(getModelsStorePath(), "metadata") 101 | } 102 | 103 | func getModelsConfigStorePath() string { 104 | return filepath.Join(getModelsStorePath(), "config") 105 | } 106 | 107 | func getModelsLayersStorePath() string { 108 | return filepath.Join(getModelsStorePath(), "layers") 109 | } 110 | 111 | func fprintf(w io.Writer, format string, a ...any) { 112 | _, _ = fmt.Fprintf(w, format, a...) 113 | } 114 | 115 | func fprint(w io.Writer, a ...any) { 116 | _, _ = fmt.Fprint(w, a...) 117 | } 118 | 119 | func tfprint(w io.Writer, border bool, headers, bodies [][]any) { 120 | tw := table.NewWriter() 121 | tw.SetOutputMirror(w) 122 | for i := range headers { 123 | tw.AppendHeader(headers[i], table.RowConfig{AutoMerge: true, AutoMergeAlign: text.AlignCenter}) 124 | } 125 | for i := range bodies { 126 | tw.AppendRow(bodies[i]) 127 | } 128 | tw.SetColumnConfigs(func() (r []table.ColumnConfig) { 129 | r = make([]table.ColumnConfig, len(headers[0])) 130 | for i := range r { 131 | r[i].Number = i + 1 132 | r[i].AutoMerge = border 133 | if len(headers) > 1 && (strings.HasPrefix(headers[1][i].(string), "Layers") || headers[1][i] == "UMA" || headers[1][i] == "NonUMA") { 134 | r[i].AutoMerge = false 135 | } 136 | r[i].Align = text.AlignCenter 137 | if !border { 138 | r[i].Align = text.AlignLeft 139 | } 140 | r[i].AlignHeader = text.AlignCenter 141 | } 142 | return r 143 | }()) 144 | { 145 | tw.Style().Options.DrawBorder = border 146 | tw.Style().Options.DrawBorder = border 147 | tw.Style().Options.SeparateHeader = border 148 | tw.Style().Options.SeparateFooter = border 149 | tw.Style().Options.SeparateColumns = border 150 | tw.Style().Options.SeparateRows = border 151 | } 152 | tw.Render() 153 | } 154 | 155 | func jprint(w io.Writer, v any) { 156 | enc := json.NewEncoder(w) 157 | enc.SetIndent("", " ") 158 | _ = enc.Encode(v) 159 | } 160 | 161 | func sprintf(format any, a ...any) string { 162 | if v, ok := format.(string); ok { 163 | if len(a) != 0 { 164 | return fmt.Sprintf(v, a...) 165 | } 166 | return v 167 | } 168 | return anyx.String(format) 169 | } 170 | 171 | func tenary(c bool, t, f any) any { 172 | if c { 173 | return t 174 | } 175 | return f 176 | } 177 | -------------------------------------------------------------------------------- /cmd/gguf-packer/pull.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "time" 12 | 13 | "github.com/awslabs/amazon-ecr-credential-helper/ecr-login" 14 | "github.com/chrismellard/docker-credential-acr-env/pkg/credhelper" 15 | "github.com/containerd/containerd/v2/pkg/archive" 16 | "github.com/google/go-containerregistry/pkg/authn" 17 | "github.com/google/go-containerregistry/pkg/authn/github" 18 | "github.com/google/go-containerregistry/pkg/crane" 19 | "github.com/google/go-containerregistry/pkg/name" 20 | conreg "github.com/google/go-containerregistry/pkg/v1" 21 | "github.com/google/go-containerregistry/pkg/v1/cache" 22 | "github.com/google/go-containerregistry/pkg/v1/google" 23 | "github.com/google/go-containerregistry/pkg/v1/remote" 24 | "github.com/google/go-containerregistry/pkg/v1/tarball" 25 | specs "github.com/gpustack/gguf-packer-go/buildkit/frontend/specs/v1" 26 | "github.com/gpustack/gguf-packer-go/util/osx" 27 | "github.com/gpustack/gguf-packer-go/util/ptr" 28 | "github.com/schollz/progressbar/v3" 29 | "github.com/spf13/cobra" 30 | ) 31 | 32 | func pull(app string) *cobra.Command { 33 | var ( 34 | insecure bool 35 | force bool 36 | ) 37 | 38 | c := &cobra.Command{ 39 | Use: "pull MODEL", 40 | Short: "Download a model from a registry.", 41 | Example: sprintf(` # Download a model 42 | %[1]s pull gpustack/qwen2:0.5b-instruct 43 | 44 | # Force download a model from remote 45 | %[1]s pull gpustack/qwen2:0.5b-instruct --force`, app), 46 | Args: cobra.ExactArgs(1), 47 | RunE: func(c *cobra.Command, args []string) (err error) { 48 | model := args[0] 49 | 50 | var cos crane.Options 51 | { 52 | co := []crane.Option{ 53 | getAuthnKeychainOption(), 54 | } 55 | if insecure { 56 | co = append(co, crane.Insecure) 57 | } 58 | cos = crane.GetOptions(co...) 59 | } 60 | 61 | rf, err := name.NewTag(model, cos.Name...) 62 | if err != nil { 63 | return fmt.Errorf("parsing model reference %q: %w", model, err) 64 | } 65 | 66 | mdp := getModelMetadataStorePath(rf) 67 | if osx.ExistsLink(mdp) && !force { 68 | return nil 69 | } 70 | 71 | var img conreg.Image 72 | { 73 | var rd *remote.Descriptor 74 | rd, err = remote.Get(rf, cos.Remote...) 75 | if err != nil { 76 | return fmt.Errorf("getting model remote %q: %w", rf.Name(), err) 77 | } 78 | img, err = retrieveOCIImage(rd) 79 | if err != nil { 80 | return err 81 | } 82 | } 83 | cfp, lsp, err := getModelConfigAndLayersStorePaths(img) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | // Link. 89 | if !force { 90 | if osx.ExistsLink(mdp) { 91 | cfpActual, err := os.Readlink(mdp) 92 | if err != nil { 93 | return fmt.Errorf("reading link %s: %w", mdp, err) 94 | } 95 | if cfpActual == cfp { 96 | return nil 97 | } 98 | // Create a tombstone. 99 | mdpTomb := filepath.Join(filepath.Dir(mdp), oldPrefix+filepath.Base(cfpActual)) 100 | if err = os.Rename(mdp, mdpTomb); err != nil { 101 | return fmt.Errorf("renaming link %s: %w", mdp, err) 102 | } 103 | // Restore a tombstone if exists. 104 | mdpTomb = filepath.Join(filepath.Dir(mdp), oldPrefix+filepath.Base(cfp)) 105 | if osx.ExistsLink(mdpTomb) { 106 | if err = os.Rename(mdpTomb, mdp); err == nil { 107 | return nil 108 | } 109 | if err = os.Remove(mdpTomb); err != nil && !os.IsNotExist(err) { 110 | return fmt.Errorf("force removing link %s: %w", mdpTomb, err) 111 | } 112 | } 113 | } 114 | } 115 | defer func() { 116 | if err != nil { 117 | return 118 | } 119 | if err = osx.ForceSymlink(cfp, mdp); err != nil { 120 | err = fmt.Errorf("link metadata %s from %s: %w", mdp, cfp, err) 121 | return 122 | } 123 | }() 124 | 125 | // Retrieve and save config. 126 | _, cfBs, err := retrieveConfigByOCIImage(img) 127 | if err != nil { 128 | return err 129 | } 130 | if err = osx.WriteFile(cfp, cfBs, 0644); err != nil { 131 | return fmt.Errorf("writing config file: %w", err) 132 | } 133 | 134 | // Extract and flatten layers. 135 | ls, err := img.Layers() 136 | if err != nil { 137 | return fmt.Errorf("retrieving image layers: %w", err) 138 | } 139 | if err = os.MkdirAll(lsp, 0755); err != nil { 140 | return fmt.Errorf("creating layers directory: %w", err) 141 | } 142 | 143 | for i := range ls { 144 | s, err := ls[i].Size() 145 | if err != nil { 146 | return fmt.Errorf("getting layer size: %w", err) 147 | } 148 | d, err := ls[i].Digest() 149 | if err != nil { 150 | return fmt.Errorf("getting layer digest: %w", err) 151 | } 152 | l, err := ls[i].Uncompressed() 153 | if err != nil { 154 | return fmt.Errorf("reading layer contents: %w", err) 155 | } 156 | pb := progressbar.NewOptions64(s, 157 | progressbar.OptionSetDescription(sprintf("[%d/%d]", i+1, len(ls))), 158 | progressbar.OptionSetWriter(c.OutOrStderr()), 159 | progressbar.OptionSetWidth(30), 160 | progressbar.OptionThrottle(65*time.Millisecond), 161 | progressbar.OptionShowBytes(true), 162 | progressbar.OptionShowCount(), 163 | progressbar.OptionSetPredictTime(false), 164 | progressbar.OptionSpinnerType(14), 165 | progressbar.OptionSetRenderBlankState(true)) 166 | if _, err = archive.Apply(c.Context(), lsp, ptr.To(progressbar.NewReader(l, pb)), archive.WithNoSameOwner()); err != nil { 167 | _ = l.Close() 168 | _ = pb.Clear() 169 | return fmt.Errorf("extracting layer %q: %w", d, err) 170 | } 171 | _ = pb.Clear() 172 | if cl, ok := l.(cacheLayerReadCloser); ok { 173 | if err = cl.Complete(); err != nil { 174 | return fmt.Errorf("completing layer %q: %w", d, err) 175 | } 176 | } 177 | } 178 | 179 | return nil 180 | }, 181 | } 182 | c.Flags().BoolVar(&insecure, "insecure", insecure, "Allow model references to be fetched without TLS.") 183 | c.Flags().BoolVar(&force, "force", force, "Always pull the model from the registry.") 184 | return c 185 | } 186 | 187 | func getAuthnKeychainOption() crane.Option { 188 | mc := authn.NewMultiKeychain( 189 | authn.DefaultKeychain, 190 | google.Keychain, 191 | github.Keychain, 192 | authn.NewKeychainFromHelper(ecr.NewECRHelper(ecr.WithLogger(io.Discard))), 193 | authn.NewKeychainFromHelper(credhelper.NewACRCredentialsHelper())) 194 | return crane.WithAuthFromKeychain(mc) 195 | } 196 | 197 | const ( 198 | dockerRegPrefix = "index.docker.io/" 199 | oldPrefix = ".old." 200 | ) 201 | 202 | func getModelMetadataStorePath(ref name.Reference) (mdp string) { 203 | const dockerRegAliasPrefix = "docker.io/" 204 | rn := ref.Name() 205 | if strings.HasPrefix(rn, dockerRegAliasPrefix) { 206 | rn = dockerRegPrefix + strings.TrimPrefix(rn, dockerRegAliasPrefix) 207 | } 208 | rn = strings.ReplaceAll(rn, ":", "/") 209 | mdp = filepath.Join(getModelsMetadataStorePath(), filepath.Clean(rn)) 210 | return mdp 211 | } 212 | 213 | func getModelConfigAndLayersStorePaths(img conreg.Image) (cfp, lsp string, err error) { 214 | cn, err := img.ConfigName() 215 | if err != nil { 216 | return "", "", fmt.Errorf("getting config name: %w", err) 217 | } 218 | cfp = filepath.Join(getModelsConfigStorePath(), cn.Algorithm, cn.Hex) 219 | lsp = filepath.Join(getModelsLayersStorePath(), cn.Algorithm, cn.Hex) 220 | return cfp, lsp, err 221 | } 222 | 223 | func retrieveOCIImage(rd *remote.Descriptor) (img conreg.Image, err error) { 224 | if rd.MediaType.IsIndex() { 225 | idx, err := rd.ImageIndex() 226 | if err != nil { 227 | return nil, fmt.Errorf("getting model index: %w", err) 228 | } 229 | idxMs, err := idx.IndexManifest() 230 | if err != nil { 231 | return nil, fmt.Errorf("getting model index manifest: %w", err) 232 | } 233 | if len(idxMs.Manifests) == 0 { 234 | return nil, errors.New("empty model index") 235 | } 236 | img, err = idx.Image(idxMs.Manifests[0].Digest) 237 | if err != nil { 238 | return nil, fmt.Errorf("getting model from index: %w", err) 239 | } 240 | } else { 241 | img, err = rd.Image() 242 | if err != nil { 243 | return nil, fmt.Errorf("getting model: %w", err) 244 | } 245 | } 246 | img = cache.Image(img, cacheLayers(getBlobsStorePath())) 247 | return img, nil 248 | } 249 | 250 | func retrieveConfigByOCIImage(img conreg.Image) (cf specs.Image, cfBs []byte, err error) { 251 | cfBs, err = img.RawConfigFile() 252 | if err != nil { 253 | return cf, cfBs, fmt.Errorf("getting config: %w", err) 254 | } 255 | if err = json.Unmarshal(cfBs, &cf); err != nil { 256 | return cf, cfBs, fmt.Errorf("unmarshalling config: %w", err) 257 | } 258 | if !isConfigAvailable(&cf) { 259 | return cf, cfBs, errors.New("unavailable model config") 260 | } 261 | return cf, cfBs, nil 262 | } 263 | 264 | func isConfigAvailable(cf *specs.Image) bool { 265 | return cf.Config.Model != nil && len(cf.Config.Model.Header.MetadataKV) != 0 && len(cf.Config.Model.TensorInfos) != 0 266 | } 267 | 268 | func getBlobsStorePath() string { 269 | return filepath.Join(storePath, "blobs") 270 | } 271 | 272 | type cacheLayers string 273 | 274 | func (c cacheLayers) Put(l conreg.Layer) (conreg.Layer, error) { 275 | digest, err := l.Digest() 276 | if err != nil { 277 | return nil, err 278 | } 279 | diffID, err := l.DiffID() 280 | if err != nil { 281 | return nil, err 282 | } 283 | cl := &cacheLayer{ 284 | Layer: l, 285 | DigestPath: c.getLayerPath(digest), 286 | DiffIDPath: c.getLayerPath(diffID), 287 | } 288 | return cl, nil 289 | } 290 | 291 | func (c cacheLayers) Get(h conreg.Hash) (conreg.Layer, error) { 292 | l, err := tarball.LayerFromFile(c.getLayerPath(h)) 293 | if os.IsNotExist(err) { 294 | return nil, cache.ErrNotFound 295 | } 296 | if errors.Is(err, io.ErrUnexpectedEOF) { 297 | if err := c.Delete(h); err != nil { 298 | return nil, err 299 | } 300 | return nil, cache.ErrNotFound 301 | } 302 | return l, err 303 | } 304 | 305 | func (c cacheLayers) Delete(h conreg.Hash) error { 306 | err := os.Remove(c.getLayerPath(h)) 307 | if os.IsNotExist(err) { 308 | return cache.ErrNotFound 309 | } 310 | return err 311 | } 312 | 313 | func (c cacheLayers) getLayerPath(h conreg.Hash) string { 314 | return filepath.Join(string(c), h.Algorithm, h.Hex) 315 | } 316 | 317 | type cacheLayer struct { 318 | conreg.Layer 319 | 320 | DigestPath string 321 | DiffIDPath string 322 | } 323 | 324 | func (c *cacheLayer) Compressed() (io.ReadCloser, error) { 325 | tmp := c.DigestPath + ".tmp" 326 | 327 | f, err := osx.CreateFile(tmp, 0700) 328 | if err != nil { 329 | return nil, err 330 | } 331 | rc, err := c.Layer.Compressed() 332 | if err != nil { 333 | return nil, err 334 | } 335 | cr := cacheLayerReadCloser{ 336 | Reader: io.TeeReader(rc, f), 337 | Closers: []io.Closer{rc, f}, 338 | Complete: func() error { return os.Rename(tmp, c.DigestPath) }, 339 | } 340 | return cr, nil 341 | } 342 | 343 | func (c *cacheLayer) Uncompressed() (io.ReadCloser, error) { 344 | tmp := c.DigestPath + ".tmp" 345 | 346 | f, err := osx.CreateFile(tmp, 0700) 347 | if err != nil { 348 | return nil, err 349 | } 350 | rc, err := c.Layer.Uncompressed() 351 | if err != nil { 352 | return nil, err 353 | } 354 | cr := cacheLayerReadCloser{ 355 | Reader: io.TeeReader(rc, f), 356 | Closers: []io.Closer{rc, f}, 357 | Complete: func() error { return os.Rename(tmp, c.DiffIDPath) }, 358 | } 359 | return cr, nil 360 | } 361 | 362 | type cacheLayerReadCloser struct { 363 | io.Reader 364 | 365 | Closers []io.Closer 366 | Complete func() error 367 | } 368 | 369 | func (c cacheLayerReadCloser) Close() (err error) { 370 | for i := range c.Closers { 371 | lastErr := c.Closers[i].Close() 372 | if err == nil { 373 | err = lastErr 374 | } 375 | } 376 | return err 377 | } 378 | -------------------------------------------------------------------------------- /cmd/gguf-packer/remove.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/fs" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "github.com/google/go-containerregistry/pkg/name" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | func remove(app string) *cobra.Command { 16 | c := &cobra.Command{ 17 | Use: "remove MODEL [MODEL...]", 18 | Short: "Remove one or more local models.", 19 | Example: sprintf(` # Remove local model by name 20 | %s remove gpustack/qwen2:0.5b-instruct 21 | 22 | # Remove local model by ID 23 | %s remove 6e76cdbc3a21`, app), 24 | Args: cobra.MinimumNArgs(1), 25 | RunE: func(c *cobra.Command, args []string) error { 26 | type queueItem struct { 27 | value string 28 | match bool 29 | err error 30 | } 31 | q := make([]queueItem, len(args)) 32 | 33 | for i := range args { 34 | if isIDAvailable(args[i]) { 35 | q[i].value = args[i] 36 | q[i].match = true 37 | continue 38 | } 39 | rf, err := name.NewTag(args[i]) 40 | if err != nil { 41 | return fmt.Errorf("parsing model reference %q: %w", args[i], err) 42 | } 43 | q[i].value = getModelMetadataStorePath(rf) 44 | } 45 | 46 | type matchItem struct { 47 | mdp string 48 | cfp string 49 | } 50 | m := map[int][]matchItem{} 51 | 52 | // Process the candidates by name. 53 | for i := range q { 54 | if q[i].match { 55 | m[i] = nil 56 | continue 57 | } 58 | rp, err := os.Readlink(q[i].value) 59 | if err != nil { 60 | q[i].err = errors.New("model not found") 61 | continue 62 | } 63 | var mdp, cfp, lsp string 64 | { 65 | mdp = q[i].value 66 | cfp = rp 67 | lsp = convertConfigStorePathToLayersStorePath(cfp) 68 | } 69 | if err = os.RemoveAll(lsp); err != nil && !os.IsNotExist(err) { 70 | q[i].err = fmt.Errorf("removing layers: %w", err) 71 | continue 72 | } 73 | if err = os.Remove(cfp); err != nil && !os.IsNotExist(err) { 74 | q[i].err = fmt.Errorf("removing config: %w", err) 75 | continue 76 | } 77 | if err = os.Remove(mdp); err != nil && !os.IsNotExist(err) { 78 | q[i].err = fmt.Errorf("removing metadata: %w", err) 79 | continue 80 | } 81 | } 82 | 83 | // Process the candidates by ID. 84 | if len(m) != 0 { 85 | msdp := getModelsMetadataStorePath() 86 | _ = filepath.Walk(msdp, func(mdp string, info fs.FileInfo, err error) error { 87 | if err != nil { 88 | return err 89 | } 90 | if info.IsDir() { 91 | if strings.HasPrefix(filepath.Base(mdp), ".") { 92 | return filepath.SkipDir 93 | } 94 | return nil 95 | } 96 | 97 | cfp, err := os.Readlink(mdp) 98 | if err != nil { 99 | // Ignore non-symbolic link. 100 | return nil 101 | } 102 | 103 | id := filepath.Base(cfp) 104 | for i := range m { 105 | if strings.HasPrefix(id, q[i].value) { 106 | m[i] = append(m[i], matchItem{mdp: mdp, cfp: cfp}) 107 | } 108 | } 109 | return nil 110 | }) 111 | for i := range m { 112 | switch s := len(m[i]); { 113 | case s == 0: 114 | q[i].err = errors.New("id not found") 115 | case s > 1: 116 | q[i].err = errors.New("id is not unique") 117 | default: 118 | var mdp, cfp, lsp string 119 | { 120 | mdp = m[i][0].mdp 121 | cfp = m[i][0].cfp 122 | lsp = convertConfigStorePathToLayersStorePath(cfp) 123 | } 124 | if err := os.RemoveAll(lsp); err != nil && !os.IsNotExist(err) { 125 | q[i].err = fmt.Errorf("removing layers: %w", err) 126 | continue 127 | } 128 | if err := os.Remove(cfp); err != nil && !os.IsNotExist(err) { 129 | q[i].err = fmt.Errorf("removing config: %w", err) 130 | continue 131 | } 132 | if err := os.Remove(mdp); err != nil && !os.IsNotExist(err) { 133 | q[i].err = fmt.Errorf("removing metadata: %w", err) 134 | continue 135 | } 136 | } 137 | } 138 | } 139 | 140 | we, wo := c.ErrOrStderr(), c.OutOrStderr() 141 | for i := range q { 142 | if err := q[i].err; err != nil { 143 | fprintf(we, "removing model %s failed: %v\n", args[i], err) 144 | continue 145 | } 146 | fprintf(wo, "removed model %s\n", args[i]) 147 | } 148 | return nil 149 | }, 150 | } 151 | return c 152 | } 153 | 154 | func convertConfigStorePathToLayersStorePath(cfp string) (lsp string) { 155 | return filepath.Join(getModelsLayersStorePath(), strings.TrimPrefix(cfp, getModelsConfigStorePath())) 156 | } 157 | 158 | func isIDAvailable(id string) bool { 159 | if len(id) < 12 || len(id) > 64 { 160 | return false 161 | } 162 | for _, c := range id { 163 | if 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { 164 | continue 165 | } 166 | return false 167 | } 168 | return true 169 | } 170 | -------------------------------------------------------------------------------- /cmd/gguf-packer/run.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "path" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/google/go-containerregistry/pkg/name" 14 | specs "github.com/gpustack/gguf-packer-go/buildkit/frontend/specs/v1" 15 | "github.com/gpustack/gguf-packer-go/util/osx" 16 | "github.com/gpustack/gguf-packer-go/util/strconvx" 17 | "github.com/gpustack/gguf-parser-go/util/stringx" 18 | "github.com/spf13/cobra" 19 | ) 20 | 21 | func run(app string) *cobra.Command { 22 | var ( 23 | by = "ghcr.io/ggerganov/llama.cpp:server" 24 | dryRun bool 25 | ) 26 | c := &cobra.Command{ 27 | Use: "run MODEL [ARG...]", 28 | Short: "Run a model by specific process, like container image or executable binary.", 29 | Example: sprintf(` # Run a model by container image: ghcr.io/ggerganov/llama.cpp:server 30 | %s run gpustack/qwen2:0.5b-instruct 31 | 32 | # Customize model running 33 | %[1]s run gpustack/qwen2:0.5b-instruct -- --port 8888 -c 8192 -np 4 34 | 35 | # Run a model by executable binary: llama-box 36 | %[1]s run gpustack/qwen2:0.5b-instruct --by llama-box 37 | 38 | # Dry run to print the command that would be executed 39 | %[1]s run gpustack/qwen2:0.5b-instruct --dry-run`, app), 40 | Args: cobra.MinimumNArgs(1), 41 | DisableFlagsInUseLine: true, 42 | FParseErrWhitelist: cobra.FParseErrWhitelist{ 43 | UnknownFlags: true, 44 | }, 45 | RunE: func(c *cobra.Command, args []string) error { 46 | isByContainer := true 47 | if _, err := name.ParseReference(by, name.StrictValidation); err != nil { 48 | isByContainer = false 49 | if !dryRun { 50 | if _, err = exec.LookPath(by); err != nil { 51 | return fmt.Errorf("looking up binary %s: %v", by, err) 52 | } 53 | } 54 | } 55 | 56 | var cfp, lsp string 57 | { 58 | model := args[0] 59 | rf, err := name.NewTag(model) 60 | if err != nil { 61 | return fmt.Errorf("parsing model reference %q: %w", model, err) 62 | } 63 | mdp := getModelMetadataStorePath(rf) 64 | if !osx.ExistsLink(mdp) { 65 | if err = pull(app).RunE(c, []string{model}); err != nil { 66 | return err 67 | } 68 | } 69 | cfp, err = os.Readlink(mdp) 70 | if err != nil { 71 | return fmt.Errorf("reading link %s: %w", mdp, err) 72 | } 73 | lsp = convertConfigStorePathToLayersStorePath(cfp) 74 | } 75 | 76 | img, err := retrieveConfigByPath(cfp) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | wdp := lsp 82 | if isByContainer { 83 | wdp = "/gp-" + stringx.RandomHex(4) 84 | } 85 | 86 | var ( 87 | cmdExec string 88 | cmdArgs []string 89 | ) 90 | 91 | if isByContainer { 92 | cmdExec = "docker" 93 | cmdArgs = []string{ 94 | "run", 95 | "--rm", 96 | "--interactive", 97 | "--tty", 98 | } 99 | if isDockerGPUSupported(c.Context()) { 100 | cmdArgs = append(cmdArgs, 101 | "--gpus", "all") 102 | } else { 103 | cmdArgs = append(cmdArgs, 104 | "--privileged") 105 | } 106 | port := "8080" 107 | for i, s := 1, len(args); i < s; i++ { 108 | if args[i] == "--port" { 109 | if i+1 >= s { 110 | return fmt.Errorf("missing value for %q", args[i]) 111 | } 112 | port = args[i+1] 113 | args = append(args[:i], args[i+2:]...) 114 | break 115 | } 116 | } 117 | cmdArgs = append(cmdArgs, 118 | "--publish", fmt.Sprintf("%s:8080", port), 119 | "--volume", fmt.Sprintf("%s:%s", lsp, wdp), 120 | by, 121 | ) 122 | } else { 123 | cmdExec = by 124 | } 125 | 126 | execArgs := img.Config.Cmd 127 | { 128 | cfg := img.Config 129 | join := filepath.Join 130 | if isByContainer { 131 | join = path.Join 132 | } 133 | for _, v := range append([]*specs.GGUFFile{cfg.Model, cfg.Drafter, cfg.Projector}, cfg.Adapters...) { 134 | if v == nil { 135 | continue 136 | } 137 | execArgs[v.CmdParameterIndex] = join(wdp, v.CmdParameterValue) 138 | } 139 | for i, s := 0, len(execArgs); i < s; i++ { 140 | if strings.HasPrefix(execArgs[i], "-") { 141 | if !strings.HasSuffix(execArgs[i], "-file") { 142 | continue 143 | } 144 | if i+1 >= s { 145 | continue 146 | } 147 | i++ 148 | execArgs[i] = join(wdp, execArgs[i]) 149 | } 150 | execArgs[i] = strconvx.Quote(execArgs[i]) 151 | } 152 | } 153 | cmdArgs = append(cmdArgs, execArgs...) 154 | 155 | cmdArgs = append(cmdArgs, args[1:]...) 156 | 157 | if isByContainer { 158 | cmdArgs = append(cmdArgs, "--host", "0.0.0.0") 159 | } 160 | 161 | if dryRun { 162 | fprintf(c.OutOrStdout(), "%s %s", cmdExec, strings.Join(cmdArgs, " ")) 163 | return nil 164 | } 165 | 166 | cmd := exec.CommandContext(c.Context(), cmdExec, cmdArgs...) 167 | cmd.Stdin = c.InOrStdin() 168 | cmd.Stdout = c.OutOrStdout() 169 | cmd.Stderr = c.ErrOrStderr() 170 | err = cmd.Run() 171 | if err != nil && strings.Contains(err.Error(), "signal: killed") { 172 | return nil 173 | } 174 | return err 175 | }, 176 | } 177 | c.Flags().StringVar(&by, "by", by, "Specify how to run the model. "+ 178 | "If given a strict format container image reference, it will be run via Docker container, "+ 179 | "otherwise it will be run via executable binary.") 180 | c.Flags().BoolVar(&dryRun, "dry-run", dryRun, "Print the command that would be executed, but do not execute it.") 181 | return c 182 | } 183 | 184 | func isDockerGPUSupported(ctx context.Context) bool { 185 | bs, err := exec. 186 | CommandContext(ctx, "docker", "info", "--format", "json"). 187 | CombinedOutput() 188 | if err != nil { 189 | return false 190 | } 191 | 192 | var r struct { 193 | Runtimes map[string]any `json:"Runtimes"` 194 | } 195 | if err = json.Unmarshal(bs, &r); err != nil { 196 | return false 197 | } 198 | 199 | _, ok := r.Runtimes["nvidia"] 200 | return ok 201 | } 202 | -------------------------------------------------------------------------------- /docs/assets/dockerhub-ollama-model-cache.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gpustack/gguf-packer-go/395fd9b5cb71da4270042b07edb18cd03f229150/docs/assets/dockerhub-ollama-model-cache.jpg -------------------------------------------------------------------------------- /examples/ggufpackerfiles/add-from-git/GGUFPackerfile: -------------------------------------------------------------------------------- 1 | ARG MODEL_NAME=Qwen2-0.5B-Instruct 2 | ARG CONVERT_TYPE=F16 3 | ARG QUANTIZE_TYPE=Q5_K_M 4 | ARG CHAT_TEMPLATE="{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" 5 | 6 | FROM scratch AS f16 7 | ADD https://huggingface.co/Qwen/${MODEL_NAME}.git ${MODEL_NAME} 8 | CONVERT --type=${CONVERT_TYPE} ${MODEL_NAME} ${MODEL_NAME}.${CONVERT_TYPE}.gguf 9 | 10 | FROM scratch 11 | LABEL gguf.model.from="Hugging Face" 12 | QUANTIZE --from=f16 --type=${QUANTIZE_TYPE} ${MODEL_NAME}.${CONVERT_TYPE}.gguf ${MODEL_NAME}.${QUANTIZE_TYPE}.gguf 13 | CAT < 0 { 57 | return defVal[0] 58 | } 59 | return v 60 | } 61 | -------------------------------------------------------------------------------- /util/osx/browser.go: -------------------------------------------------------------------------------- 1 | package osx 2 | 3 | import ( 4 | "errors" 5 | "os/exec" 6 | "runtime" 7 | ) 8 | 9 | // OpenBrowser opens the default web browser to the specified URL. 10 | func OpenBrowser(url string) error { 11 | switch runtime.GOOS { 12 | case "linux": 13 | return exec.Command("xdg-open", url).Start() 14 | case "darwin": 15 | return exec.Command("open", url).Start() 16 | case "windows": 17 | return exec.Command("cmd", "/c", "start", url).Start() 18 | default: 19 | } 20 | return errors.New("unsupported platform") 21 | } 22 | -------------------------------------------------------------------------------- /util/osx/env.go: -------------------------------------------------------------------------------- 1 | package osx 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | // ExistEnv checks if the environment variable named by the key exists. 8 | func ExistEnv(key string) bool { 9 | _, ok := os.LookupEnv(key) 10 | return ok 11 | } 12 | 13 | // Getenv retrieves the value of the environment variable named by the key. 14 | // It returns the default, which will be empty if the variable is not present. 15 | // To distinguish between an empty value and an unset value, use LookupEnv. 16 | func Getenv(key string, def ...string) string { 17 | e, ok := os.LookupEnv(key) 18 | if !ok && len(def) != 0 { 19 | return def[0] 20 | } 21 | 22 | return e 23 | } 24 | 25 | // ExpandEnv is similar to Getenv, 26 | // but replaces ${var} or $var in the result. 27 | func ExpandEnv(key string, def ...string) string { 28 | return os.ExpandEnv(Getenv(key, def...)) 29 | } 30 | -------------------------------------------------------------------------------- /util/osx/file.go: -------------------------------------------------------------------------------- 1 | package osx 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | ) 10 | 11 | // InlineTilde replaces the leading ~ with the home directory. 12 | func InlineTilde(path string) string { 13 | if path == "" { 14 | return path 15 | } 16 | if strings.HasPrefix(path, "~"+string(filepath.Separator)) { 17 | hd, err := os.UserHomeDir() 18 | if err == nil { 19 | path = filepath.Join(hd, path[2:]) 20 | } 21 | } 22 | return path 23 | } 24 | 25 | // Open is similar to os.Open but supports ~ as the home directory. 26 | func Open(path string) (*os.File, error) { 27 | p := filepath.Clean(path) 28 | p = InlineTilde(p) 29 | return os.Open(p) 30 | } 31 | 32 | // Exists checks if the given path exists. 33 | func Exists(path string, checks ...func(os.FileInfo) bool) bool { 34 | p := filepath.Clean(path) 35 | p = InlineTilde(p) 36 | 37 | stat, err := os.Lstat(p) 38 | if err != nil { 39 | return false 40 | } 41 | 42 | for i := range checks { 43 | if checks[i] == nil { 44 | continue 45 | } 46 | 47 | if !checks[i](stat) { 48 | return false 49 | } 50 | } 51 | 52 | return true 53 | } 54 | 55 | // ExistsDir checks if the given path exists and is a directory. 56 | func ExistsDir(path string) bool { 57 | return Exists(path, func(stat os.FileInfo) bool { 58 | return stat.Mode().IsDir() 59 | }) 60 | } 61 | 62 | // ExistsLink checks if the given path exists and is a symbolic link. 63 | func ExistsLink(path string) bool { 64 | return Exists(path, func(stat os.FileInfo) bool { 65 | return stat.Mode()&os.ModeSymlink != 0 66 | }) 67 | } 68 | 69 | // ExistsFile checks if the given path exists and is a regular file. 70 | func ExistsFile(path string) bool { 71 | return Exists(path, func(stat os.FileInfo) bool { 72 | return stat.Mode().IsRegular() 73 | }) 74 | } 75 | 76 | // ExistsSocket checks if the given path exists and is a socket. 77 | func ExistsSocket(path string) bool { 78 | return Exists(path, func(stat os.FileInfo) bool { 79 | return stat.Mode()&os.ModeSocket != 0 80 | }) 81 | } 82 | 83 | // ExistsDevice checks if the given path exists and is a device. 84 | func ExistsDevice(path string) bool { 85 | return Exists(path, func(stat os.FileInfo) bool { 86 | return stat.Mode()&os.ModeDevice != 0 87 | }) 88 | } 89 | 90 | // Close closes the given io.Closer without error. 91 | func Close(c io.Closer) { 92 | if c == nil { 93 | return 94 | } 95 | _ = c.Close() 96 | } 97 | 98 | // WriteFile is similar to os.WriteFile but supports ~ as the home directory, 99 | // and also supports the parent directory creation. 100 | func WriteFile(name string, data []byte, perm os.FileMode) error { 101 | p := filepath.Clean(name) 102 | p = InlineTilde(p) 103 | 104 | if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { 105 | return err 106 | } 107 | 108 | return os.WriteFile(p, data, perm) 109 | } 110 | 111 | // CreateFile is similar to os.Create but supports ~ as the home directory, 112 | // and also supports the parent directory creation. 113 | func CreateFile(name string, perm os.FileMode) (*os.File, error) { 114 | p := filepath.Clean(name) 115 | p = InlineTilde(p) 116 | 117 | if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { 118 | return nil, err 119 | } 120 | 121 | return os.OpenFile(p, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm) 122 | } 123 | 124 | // OpenFile is similar to os.OpenFile but supports ~ as the home directory, 125 | // and also supports the parent directory creation. 126 | func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { 127 | p := filepath.Clean(name) 128 | p = InlineTilde(p) 129 | 130 | if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { 131 | return nil, err 132 | } 133 | 134 | return os.OpenFile(p, flag, perm) 135 | } 136 | 137 | // Symlink is similar to os.Symlink but supports ~ as the home directory, 138 | // and also supports the parent directory creation. 139 | func Symlink(oldname, newname string) error { 140 | op, np := filepath.Clean(oldname), filepath.Clean(newname) 141 | op, np = InlineTilde(op), InlineTilde(np) 142 | 143 | if err := os.MkdirAll(filepath.Dir(op), 0o700); err != nil { 144 | return err 145 | } 146 | if err := os.MkdirAll(filepath.Dir(np), 0o700); err != nil { 147 | return err 148 | } 149 | 150 | return os.Symlink(oldname, newname) 151 | } 152 | 153 | func ForceSymlink(oldname, newname string) error { 154 | np := filepath.Clean(newname) 155 | np = InlineTilde(np) 156 | 157 | if err := os.Remove(np); err != nil && !os.IsNotExist(err) { 158 | return fmt.Errorf("removing destination %s: %w", np, err) 159 | } 160 | 161 | return Symlink(oldname, np) 162 | } 163 | -------------------------------------------------------------------------------- /util/osx/file_mmap.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package osx 15 | 16 | import ( 17 | "errors" 18 | "fmt" 19 | "io" 20 | "os" 21 | "path/filepath" 22 | "runtime/debug" 23 | "syscall" 24 | ) 25 | 26 | type MmapFile struct { 27 | f *os.File 28 | b []byte 29 | } 30 | 31 | func OpenMmapFile(path string) (*MmapFile, error) { 32 | return OpenMmapFileWithSize(path, 0) 33 | } 34 | 35 | func OpenMmapFileWithSize(path string, size int) (*MmapFile, error) { 36 | p := filepath.Clean(path) 37 | p = InlineTilde(p) 38 | 39 | f, err := os.Open(p) 40 | if err != nil { 41 | return nil, fmt.Errorf("try lock file: %w", err) 42 | } 43 | if size <= 0 { 44 | info, err := f.Stat() 45 | if err != nil { 46 | Close(f) 47 | return nil, fmt.Errorf("stat: %w", err) 48 | } 49 | size = int(info.Size()) 50 | } 51 | 52 | b, err := mmap(f, size) 53 | if err != nil { 54 | Close(f) 55 | return nil, fmt.Errorf("mmap, size %d: %w", size, err) 56 | } 57 | 58 | return &MmapFile{f: f, b: b}, nil 59 | } 60 | 61 | func (f *MmapFile) Close() error { 62 | err0 := munmap(f.b) 63 | err1 := f.f.Close() 64 | 65 | if err0 != nil { 66 | return err0 67 | } 68 | return err1 69 | } 70 | 71 | func (f *MmapFile) Bytes() []byte { 72 | return f.b 73 | } 74 | 75 | func (f *MmapFile) Len() int64 { 76 | return int64(len(f.b)) 77 | } 78 | 79 | var ErrPageFault = errors.New("page fault occurred while reading from memory map") 80 | 81 | func (f *MmapFile) ReadAt(p []byte, off int64) (_ int, err error) { 82 | if off < 0 { 83 | return 0, syscall.EINVAL 84 | } 85 | if off > f.Len() { 86 | return 0, io.EOF 87 | } 88 | 89 | old := debug.SetPanicOnFault(true) 90 | defer func() { 91 | debug.SetPanicOnFault(old) 92 | if recover() != nil { 93 | err = ErrPageFault 94 | } 95 | }() 96 | 97 | n := copy(p, f.b[off:]) 98 | if n < len(p) { 99 | err = io.EOF 100 | } 101 | return n, err 102 | } 103 | -------------------------------------------------------------------------------- /util/osx/file_mmap_js.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package osx 15 | 16 | import ( 17 | "errors" 18 | "os" 19 | ) 20 | 21 | func mmap(f *os.File, length int) ([]byte, error) { 22 | return nil, errors.New("unsupported") 23 | } 24 | 25 | func munmap(b []byte) (err error) { 26 | return errors.New("unsupported") 27 | } 28 | -------------------------------------------------------------------------------- /util/osx/file_mmap_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | //go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris 15 | 16 | package osx 17 | 18 | import ( 19 | "os" 20 | 21 | "golang.org/x/sys/unix" 22 | ) 23 | 24 | func mmap(f *os.File, length int) ([]byte, error) { 25 | return unix.Mmap(int(f.Fd()), 0, length, unix.PROT_READ, unix.MAP_SHARED) 26 | } 27 | 28 | func munmap(b []byte) (err error) { 29 | return unix.Munmap(b) 30 | } 31 | -------------------------------------------------------------------------------- /util/osx/file_mmap_windows.go: -------------------------------------------------------------------------------- 1 | package osx 2 | 3 | import ( 4 | "os" 5 | "syscall" 6 | "unsafe" 7 | ) 8 | 9 | func mmap(f *os.File, size int) ([]byte, error) { 10 | low, high := uint32(size), uint32(size>>32) 11 | h, errno := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, high, low, nil) 12 | if h == 0 { 13 | return nil, os.NewSyscallError("CreateFileMapping", errno) 14 | } 15 | 16 | addr, errno := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, uintptr(size)) 17 | if addr == 0 { 18 | return nil, os.NewSyscallError("MapViewOfFile", errno) 19 | } 20 | 21 | if err := syscall.CloseHandle(h); err != nil { 22 | return nil, os.NewSyscallError("CloseHandle", err) 23 | } 24 | 25 | return (*[maxMapSize]byte)(unsafe.Pointer(uintptr(addr)))[:size], nil 26 | } 27 | 28 | func munmap(b []byte) error { 29 | if err := syscall.UnmapViewOfFile((uintptr)(unsafe.Pointer(&b[0]))); err != nil { 30 | return os.NewSyscallError("UnmapViewOfFile", err) 31 | } 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /util/osx/file_mmap_windows_386.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package osx 15 | 16 | const maxMapSize = 0x7FFFFFFF // 2GB 17 | -------------------------------------------------------------------------------- /util/osx/file_mmap_windows_non386.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | //go:build windows && !386 15 | 16 | package osx 17 | 18 | const maxMapSize = 0xFFFFFFFFFFFF // 256TB 19 | -------------------------------------------------------------------------------- /util/osx/homedir.go: -------------------------------------------------------------------------------- 1 | package osx 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "time" 7 | ) 8 | 9 | // UserHomeDir is similar to os.UserHomeDir, 10 | // but returns the temp dir if the home dir is not found. 11 | func UserHomeDir() string { 12 | hd, err := os.UserHomeDir() 13 | if err != nil { 14 | hd = filepath.Join(os.TempDir(), time.Now().Format(time.DateOnly)) 15 | } 16 | return hd 17 | } 18 | -------------------------------------------------------------------------------- /util/ptr/pointer.go: -------------------------------------------------------------------------------- 1 | package ptr 2 | 3 | func To[T any](v T) *T { 4 | return &v 5 | } 6 | 7 | func From[T any](ptr *T, def T) T { 8 | if ptr != nil { 9 | return *ptr 10 | } 11 | return def 12 | } 13 | 14 | func Equal[T comparable](a, b *T) bool { 15 | if a != nil && b != nil { 16 | return *a == *b 17 | } 18 | return false 19 | } 20 | -------------------------------------------------------------------------------- /util/signalx/handler.go: -------------------------------------------------------------------------------- 1 | package signalx 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | ) 8 | 9 | var registered = make(chan struct{}) 10 | 11 | // Handler registers for signals and returns a context. 12 | func Handler() context.Context { 13 | close(registered) // Panics when called twice. 14 | 15 | sigChan := make(chan os.Signal, len(sigs)) 16 | ctx, cancel := context.WithCancel(context.Background()) 17 | 18 | // Register for signals. 19 | signal.Notify(sigChan, sigs...) 20 | 21 | // Process signals. 22 | go func() { 23 | var exited bool 24 | for range sigChan { 25 | if exited { 26 | os.Exit(1) 27 | } 28 | cancel() 29 | exited = true 30 | } 31 | }() 32 | 33 | return ctx 34 | } 35 | -------------------------------------------------------------------------------- /util/signalx/handler_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package signalx 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | ) 9 | 10 | var sigs = []os.Signal{syscall.SIGINT, syscall.SIGTERM} 11 | -------------------------------------------------------------------------------- /util/signalx/handler_windows.go: -------------------------------------------------------------------------------- 1 | package signalx 2 | 3 | import ( 4 | "os" 5 | "syscall" 6 | ) 7 | 8 | var sigs = []os.Signal{syscall.SIGINT} 9 | -------------------------------------------------------------------------------- /util/strconvx/quote.go: -------------------------------------------------------------------------------- 1 | package strconvx 2 | 3 | import ( 4 | "strconv" 5 | "unicode" 6 | ) 7 | 8 | // ShouldQuote returns true if the string should be quoted. 9 | func ShouldQuote(s string) bool { 10 | if len(s) == 0 { 11 | return true 12 | } 13 | for _, r := range s { 14 | if unicode.IsSpace(r) || r == '"' || r == '\'' || r == '\\' { 15 | return true 16 | } 17 | } 18 | return false 19 | } 20 | 21 | // Quote is similar to strconv.Quote, 22 | // but it only quotes the string if it contains spaces or special characters. 23 | func Quote(s string) string { 24 | if !ShouldQuote(s) { 25 | return s 26 | } 27 | return strconv.Quote(s) 28 | } 29 | --------------------------------------------------------------------------------