├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .githooks └── pre-commit ├── .github └── workflows │ ├── ci-build.yaml │ └── release.yaml ├── .gitignore ├── .gitpod └── automations.yaml ├── .goreleaser.yaml ├── .theia └── launch.json ├── .vscode └── settings.json ├── BUILD.yaml ├── LICENSE ├── README.md ├── WORKSPACE.yaml ├── cmd ├── bashCompletion.go ├── build.go ├── collect.go ├── describe-const.go ├── describe-dependants.go ├── describe-dependencies.go ├── describe-environment-manifest.go ├── describe-gitinfo.go ├── describe-manifest.go ├── describe-script.go ├── describe-tree.go ├── describe.go ├── exec.go ├── experiemental-unmount.go ├── experimental-mount.go ├── export.go ├── fmt.go ├── init.go ├── link.go ├── plumbing-exec.go ├── plumbing.go ├── provenance-assert.go ├── provenance-export.go ├── provenance.go ├── root.go ├── run.go ├── sbom-export.go ├── sbom-scan.go ├── sbom.go ├── version.go └── vet.go ├── codecov.yml ├── go.mod ├── go.sum ├── logo.png ├── main.go └── pkg ├── doublestar ├── doublestar.go └── doublestar_test.go ├── leeway ├── build.go ├── build_darwin.go ├── build_internal_test.go ├── build_linux.go ├── build_test.go ├── cache │ ├── local │ │ ├── fs.go │ │ └── fs_test.go │ ├── remote │ │ ├── gsutil.go │ │ ├── no_cache.go │ │ ├── no_cache_test.go │ │ ├── s3.go │ │ ├── s3_download_test.go │ │ └── s3_test.go │ ├── testing │ │ └── mock.go │ └── types.go ├── compression.go ├── container_image.go ├── export.go ├── fixtures │ └── load-workspace.yaml ├── format.go ├── format_test.go ├── gitinfo.go ├── gitinfo_test.go ├── package.go ├── package_test.go ├── provenance.go ├── reporter.go ├── reporter_test.go ├── sbom-scan.go ├── sbom.go ├── scripts.go ├── scripts_test.go ├── version.go ├── watcher.go ├── workspace.go └── workspace_test.go ├── linker ├── golang.go └── yarn.go ├── prettyprint └── prettyprint.go ├── provutil ├── access.go └── assert.go ├── testutil ├── testutil.go └── testutil_test.go └── vet ├── components.go ├── docker.go ├── docker_test.go ├── generic.go ├── golang.go ├── packages.go ├── vet.go └── yarn.go /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/devcontainers/base:ubuntu 2 | 3 | # Install leeway build tool 4 | RUN bash -c 'set -euo pipefail && \ 5 | LATEST_LEEWAY_VERSION=$(curl -s https://api.github.com/repos/gitpod-io/leeway/releases/latest | jq -r ".tag_name" | sed "s/^v//") && \ 6 | echo "Installing leeway version: $LATEST_LEEWAY_VERSION" && \ 7 | curl -L -o /tmp/leeway.tar.gz "https://github.com/gitpod-io/leeway/releases/download/v${LATEST_LEEWAY_VERSION}/leeway_Linux_x86_64.tar.gz" && \ 8 | tar -xzf /tmp/leeway.tar.gz -C /tmp && \ 9 | install -m 755 /tmp/leeway /usr/local/bin/ && \ 10 | rm /tmp/leeway.tar.gz /tmp/leeway' 11 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "leeway", 3 | "build": { 4 | "dockerfile": "Dockerfile" 5 | }, 6 | "features": { 7 | "ghcr.io/devcontainers/features/node:1": { 8 | "version": "18" 9 | }, 10 | "ghcr.io/devcontainers/features/go:1": { 11 | "version": "1.24" 12 | }, 13 | "ghcr.io/devcontainers/features/common-utils:2": {}, 14 | "ghcr.io/dhoeric/features/google-cloud-cli:1": {}, 15 | "ghcr.io/devcontainers/features/aws-cli:1": {} 16 | }, 17 | "forwardPorts": [8080, 3000, 9000], 18 | "portsAttributes": { 19 | "8080": { 20 | "label": "Application Server", 21 | "onAutoForward": "notify" 22 | }, 23 | "3000": { 24 | "label": "Development Server", 25 | "onAutoForward": "notify" 26 | }, 27 | "9000": { 28 | "label": "Debug Server", 29 | "onAutoForward": "notify" 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /.githooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | go test -v ./... -------------------------------------------------------------------------------- /.github/workflows/ci-build.yaml: -------------------------------------------------------------------------------- 1 | name: Unit tests 2 | on: 3 | push: 4 | branches: 5 | - 'main' 6 | - 'v*' 7 | pull_request: 8 | branches: 9 | - 'main' 10 | 11 | env: 12 | GO_VERSION: '1.24' 13 | 14 | jobs: 15 | check-go: 16 | name: Check Go code 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout code 20 | uses: actions/checkout@v4 21 | - name: Setup Golang 22 | uses: actions/setup-go@v5 23 | with: 24 | go-version: ${{ env.GO_VERSION }} 25 | cache: true 26 | - name: Check for tidyness of go.mod and go.sum 27 | run: | 28 | go mod tidy 29 | git diff --exit-code -- . 30 | 31 | build-go: 32 | name: Build & cache Go code 33 | runs-on: ubuntu-latest 34 | steps: 35 | - name: Checkout code 36 | uses: actions/checkout@v4 37 | - name: Setup Golang 38 | uses: actions/setup-go@v5 39 | with: 40 | go-version: ${{ env.GO_VERSION }} 41 | cache: true 42 | 43 | lint-go: 44 | name: Lint Go code 45 | runs-on: ubuntu-latest 46 | steps: 47 | - name: Checkout code 48 | uses: actions/checkout@v4 49 | - name: Run golangci-lint 50 | uses: golangci/golangci-lint-action@v4 51 | with: 52 | args: --timeout 5m 53 | 54 | test-go: 55 | name: Run unit tests 56 | runs-on: ubuntu-latest 57 | needs: 58 | - build-go 59 | steps: 60 | - name: Create checkout directory 61 | run: mkdir -p ~/go/src/github.com/gitpod-io 62 | - name: Checkout code 63 | uses: actions/checkout@v4 64 | - name: Create symlink in GOPATH 65 | run: ln -s $(pwd) ~/go/src/github.com/gitpod-io/leeway 66 | - name: Setup Golang 67 | uses: actions/setup-go@v5 68 | with: 69 | go-version: ${{ env.GO_VERSION }} 70 | cache: true 71 | - name: Run all unit tests 72 | run: go test -v -coverprofile=coverage.out $(go list ./...) 73 | - name: Generate code coverage artifacts 74 | uses: actions/upload-artifact@v4 75 | with: 76 | name: code-coverage 77 | path: coverage.out 78 | 79 | test-go-race: 80 | name: Run unit tests with -race 81 | runs-on: ubuntu-latest 82 | needs: 83 | - build-go 84 | steps: 85 | - name: Create checkout directory 86 | run: mkdir -p ~/go/src/github.com/gitpod-io 87 | - name: Checkout code 88 | uses: actions/checkout@v4 89 | - name: Create symlink in GOPATH 90 | run: ln -s $(pwd) ~/go/src/github.com/gitpod-io/leeway 91 | - name: Setup Golang 92 | uses: actions/setup-go@v5 93 | with: 94 | go-version: ${{ env.GO_VERSION }} 95 | cache: true 96 | - name: Run all unit tests 97 | run: go test -race -v $(go list ./...) 98 | 99 | analyze: 100 | name: Process & analyze test artifacts 101 | runs-on: ubuntu-latest 102 | needs: 103 | - test-go 104 | steps: 105 | - name: Checkout code 106 | uses: actions/checkout@v4 107 | with: 108 | fetch-depth: 10 109 | - name: Create test-results directory 110 | run: | 111 | mkdir -p test-results 112 | - name: Get code coverage artifiact 113 | uses: actions/download-artifact@v4 114 | with: 115 | name: code-coverage 116 | - name: Upload code coverage information to codecov.io 117 | uses: codecov/codecov-action@v4 118 | with: 119 | file: coverage.out 120 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release Go project 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" # triggers only if push new tag version, like `0.8.4` or else 7 | 8 | permissions: 9 | contents: write 10 | 11 | jobs: 12 | release: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@v4 17 | with: 18 | fetch-depth: 0 19 | - name: Setup Golang 20 | uses: actions/setup-go@v5 21 | with: 22 | go-version: "1.24" 23 | - name: Docker Login 24 | uses: docker/login-action@v1 25 | with: 26 | registry: ghcr.io 27 | username: ${{ github.repository_owner }} 28 | password: ${{ secrets.GITHUB_TOKEN }} 29 | - name: Run GoReleaser 30 | uses: goreleaser/goreleaser-action@v6 31 | if: startsWith(github.ref, 'refs/tags/') 32 | with: 33 | distribution: goreleaser 34 | version: "~> v2" 35 | args: release --clean 36 | env: 37 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | rice-box.go 3 | coverage.out 4 | .idea 5 | 6 | dist/ 7 | -------------------------------------------------------------------------------- /.gitpod/automations.yaml: -------------------------------------------------------------------------------- 1 | tasks: 2 | install-deps: 3 | name: Install Go Dependencies 4 | description: Download and cache Go module dependencies 5 | command: | 6 | echo "Installing Go dependencies..." 7 | go mod download 8 | echo "Go dependencies installed successfully" 9 | triggeredBy: 10 | - postDevcontainerStart 11 | 12 | build-leeway: 13 | name: Build Leeway Application 14 | description: Build the main leeway application 15 | command: | 16 | echo "Building leeway application..." 17 | go build -o bin/leeway . 18 | echo "Leeway application built successfully" 19 | dependsOn: 20 | - install-deps 21 | 22 | test: 23 | name: Run Tests 24 | description: Run Go tests for the project 25 | command: | 26 | echo "Running tests..." 27 | go test ./... 28 | echo "Tests completed" 29 | dependsOn: 30 | - install-deps 31 | triggeredBy: 32 | - manual 33 | 34 | lint: 35 | name: Run Linter 36 | description: Run golangci-lint on the codebase 37 | command: | 38 | echo "Running linter..." 39 | if command -v golangci-lint >/dev/null 2>&1; then 40 | golangci-lint run 41 | else 42 | echo "golangci-lint not available, skipping lint check" 43 | fi 44 | echo "Linting completed" 45 | dependsOn: 46 | - install-deps 47 | triggeredBy: 48 | - manual 49 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | before: 4 | hooks: 5 | - go generate ./... 6 | 7 | builds: 8 | - env: 9 | - CGO_ENABLED=0 10 | goos: 11 | - linux 12 | - darwin 13 | ldflags: 14 | - -s -w -X github.com/gitpod-io/leeway/pkg/leeway.Version={{.Version}}-{{.ShortCommit}} 15 | goarch: 16 | - amd64 17 | - arm64 18 | 19 | archives: 20 | - format: tar.gz 21 | name_template: >- 22 | {{ .ProjectName }}_ 23 | {{- title .Os }}_ 24 | {{- if eq .Arch "amd64" }}x86_64 25 | {{- else }}{{ .Arch }}{{ end }} 26 | {{- if .Arm }}v{{ .Arm }}{{ end }} 27 | 28 | changelog: 29 | sort: asc 30 | filters: 31 | exclude: 32 | - "^docs:" 33 | - "^test:" 34 | -------------------------------------------------------------------------------- /.theia/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | "version": "0.2.0", 5 | "configurations": [ 6 | { 7 | "name": "Launch", 8 | "type": "go", 9 | "request": "launch", 10 | "mode": "auto", 11 | "program": "${workspaceFolder}/main.go", 12 | "env": { 13 | "LEEWAY_EXPERIMENTAL": "true" 14 | }, 15 | "args": ["fmt", "fixtures/nested-ws/pkg0/BUILD.yaml"] 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "go.lintTool": "golangci-lint" 3 | } -------------------------------------------------------------------------------- /BUILD.yaml: -------------------------------------------------------------------------------- 1 | const: 2 | foo: bar 3 | packages: 4 | - name: helloworld 5 | type: generic 6 | ephemeral: true 7 | config: 8 | commands: 9 | - ["echo", "hello world"] 10 | - name: app 11 | type: go 12 | srcs: 13 | - "**/*.go" 14 | - "**/*.html" 15 | - "**/*.js" 16 | - go.mod 17 | - go.sum 18 | deps: 19 | - :helloworld 20 | argdeps: 21 | - version 22 | prep: 23 | - ["echo", "hello prep"] 24 | config: 25 | dontTest: true 26 | buildArgs: 27 | - -ldflags 28 | - -X main.version=${version} 29 | library: false 30 | - name: goreleaser 31 | type: generic 32 | config: 33 | commands: 34 | - ["curl", "-OL", "https://github.com/goreleaser/goreleaser/releases/download/v1.10.3/goreleaser_Linux_x86_64.tar.gz"] 35 | - ["tar", "xzf", "goreleaser_Linux_x86_64.tar.gz"] 36 | - ["rm", "goreleaser_Linux_x86_64.tar.gz"] 37 | scripts: 38 | - name: release 39 | description: releases leeway to GitHub 40 | deps: 41 | - :goreleaser 42 | script: | 43 | if [ "A$GITHUB_TOKEN" = "A" ]; then 44 | echo "Please set the GITHUB_TOKEN env var" 45 | exit 1 46 | fi 47 | 48 | if [ $(git tag | grep ${version}) ]; then 49 | git tag --delete ${version} 50 | git push origin --delete ${version} || echo "version wasn't pushed yet - that's ok" 51 | fi 52 | 53 | git tag ${version} 54 | git push origin ${version} 55 | goreleaser --rm-dist 56 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 TypeFox GmbH 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /WORKSPACE.yaml: -------------------------------------------------------------------------------- 1 | defaultTarget: "//:app" 2 | defaultArgs: 3 | version: dev 4 | environmentManifest: 5 | - name: "node" 6 | command: ["node", "--version"] 7 | - name: "yarn" 8 | command: ["yarn", "--version"] 9 | provenance: 10 | enabled: true 11 | slsa: true 12 | sbom: 13 | enabled: true 14 | scanVulnerabilities: true 15 | # failOn: ["critical", "high"] 16 | # ignoreVulnerabilities: 17 | # - vulnerability: GHSA-265r-hfxg-fhmg 18 | # reason: "Not exploitable in our context" 19 | variants: 20 | - name: nogit 21 | srcs: 22 | exclude: 23 | - "**/.git" 24 | -------------------------------------------------------------------------------- /cmd/bashCompletion.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | // bashCompletionCmd represents the bashCompletion command 10 | var bashCompletionCmd = &cobra.Command{ 11 | Use: "bash-completion", 12 | Short: "Provides bash completion for leeway. Use with `. <(leeway bash-completion)`", 13 | Hidden: true, 14 | RunE: func(cmd *cobra.Command, args []string) error { 15 | return rootCmd.GenBashCompletion(os.Stdout) 16 | }, 17 | } 18 | 19 | func init() { 20 | rootCmd.AddCommand(bashCompletionCmd) 21 | } 22 | -------------------------------------------------------------------------------- /cmd/collect.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "sort" 5 | "strings" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | 10 | "github.com/gitpod-io/leeway/pkg/leeway" 11 | "github.com/gitpod-io/leeway/pkg/prettyprint" 12 | ) 13 | 14 | type fileDescription struct { 15 | Name string `json:"name" yaml:"name"` 16 | Version string `json:"version" yaml:"version"` 17 | Package string `json:"package" yaml:"package"` 18 | } 19 | 20 | type variantDescription struct { 21 | Name string `json:"name" yaml:"name"` 22 | Sources struct { 23 | Include []string `json:"include" yaml:"include"` 24 | Exclude []string `json:"exclude" yaml:"exclude"` 25 | } `json:"srcs" yaml:"srcs"` 26 | Environment []string `json:"env" yaml:"env"` 27 | Config map[leeway.PackageType]configDescription `json:"config" yaml:"config"` 28 | } 29 | 30 | // collectCmd represents the collect command 31 | var collectCmd = &cobra.Command{ 32 | Use: "collect [components|packages|scripts|files]", 33 | Short: "Collects all packages in a workspace", 34 | Args: cobra.MatchAll(cobra.OnlyValidArgs, cobra.MaximumNArgs(1)), 35 | ValidArgs: []string{"components", "packages", "scripts", "scripts", "files"}, 36 | Run: func(cmd *cobra.Command, args []string) { 37 | workspace, err := getWorkspace() 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | 42 | var tpe string 43 | if len(args) == 0 { 44 | tpe = "packages" 45 | } else { 46 | tpe = args[0] 47 | } 48 | 49 | selectStr, _ := cmd.Flags().GetString("select") 50 | var selector func(c *leeway.Component) bool 51 | segs := strings.Split(selectStr, "=") 52 | if len(selectStr) == 0 { 53 | selector = func(c *leeway.Component) bool { 54 | return true 55 | } 56 | } else if len(segs) == 1 { 57 | selector = func(c *leeway.Component) bool { 58 | _, ok := c.Constants[segs[0]] 59 | return ok 60 | } 61 | } else if len(segs) == 2 { 62 | selector = func(c *leeway.Component) bool { 63 | return c.Constants[segs[0]] == segs[1] 64 | } 65 | } else { 66 | log.Fatal("selector must either be a constant name or const=value") 67 | } 68 | 69 | w := getWriterFromFlags(cmd) 70 | switch tpe { 71 | case "components": 72 | if w.Format == prettyprint.TemplateFormat && w.FormatString == "" { 73 | w.FormatString = `{{ range . }}{{ .Name }}{{"\n"}}{{ end }}` 74 | } 75 | decs := make([]componentDescription, 0, len(workspace.Components)) 76 | for _, comp := range workspace.Components { 77 | if !selector(comp) { 78 | continue 79 | } 80 | decs = append(decs, newComponentDescription(comp)) 81 | } 82 | sort.Slice(decs, func(i, j int) bool { return decs[i].Name < decs[j].Name }) 83 | err = w.Write(decs) 84 | if err != nil { 85 | log.Fatal(err) 86 | } 87 | case "packages": 88 | if w.Format == prettyprint.TemplateFormat && w.FormatString == "" { 89 | w.FormatString = `{{ range . }}{{ .Metadata.FullName }}{{"\t"}}{{ .Metadata.Version }}{{"\n"}}{{ end }}` 90 | } 91 | decs := make([]packageDescription, 0, len(workspace.Packages)) 92 | for _, pkg := range workspace.Packages { 93 | if !selector(pkg.C) { 94 | continue 95 | } 96 | 97 | decs = append(decs, newPackageDesription(pkg)) 98 | } 99 | sort.Slice(decs, func(i, j int) bool { return decs[i].Metadata.FullName < decs[j].Metadata.FullName }) 100 | err = w.Write(decs) 101 | if err != nil { 102 | log.Fatal(err) 103 | } 104 | case "scripts": 105 | if w.Format == prettyprint.TemplateFormat && w.FormatString == "" { 106 | w.FormatString = `{{ range . }}{{ .FullName }}{{ if .Description }}{{"\t"}}{{ .Description }}{{ end }}{{"\n"}}{{ end }}` 107 | } 108 | decs := make([]scriptDescription, 0, len(workspace.Scripts)) 109 | for _, scr := range workspace.Scripts { 110 | if !selector(scr.C) { 111 | continue 112 | } 113 | 114 | decs = append(decs, newScriptDescription(scr)) 115 | } 116 | sort.Slice(decs, func(i, j int) bool { return decs[i].FullName < decs[j].FullName }) 117 | err = w.Write(decs) 118 | if err != nil { 119 | log.Fatal(err) 120 | } 121 | case "files": 122 | if w.Format == prettyprint.TemplateFormat && w.FormatString == "" { 123 | w.FormatString = `{{ range . }}{{ .Name }}{{"\t"}}{{ .Version }}{{"\n"}}{{ end }}` 124 | } 125 | decs := make([]fileDescription, 0, len(workspace.Packages)) 126 | for _, pkg := range workspace.Packages { 127 | if !selector(pkg.C) { 128 | continue 129 | } 130 | 131 | pkgn := pkg.FullName() 132 | mf, err := pkg.ContentManifest() 133 | if err != nil { 134 | log.Fatal(err) 135 | } 136 | fs := make([]fileDescription, len(mf)) 137 | for i, f := range mf { 138 | segs := strings.Split(f, ":") 139 | fs[i] = fileDescription{Name: segs[0], Version: segs[1], Package: pkgn} 140 | } 141 | 142 | decs = append(decs, fs...) 143 | } 144 | sort.Slice(decs, func(i, j int) bool { return decs[i].Name < decs[j].Name }) 145 | err = w.Write(decs) 146 | if err != nil { 147 | log.Fatal(err) 148 | } 149 | case "variants": 150 | if w.Format == prettyprint.TemplateFormat && w.FormatString == "" { 151 | w.FormatString = `{{ range . }}{{ .Name }}{{"\n"}}{{ end }}` 152 | } 153 | decs := make([]variantDescription, len(workspace.Variants)) 154 | for i, v := range workspace.Variants { 155 | decs[i] = variantDescription{ 156 | Name: v.Name, 157 | Environment: v.Environment, 158 | Config: make(map[leeway.PackageType]configDescription), 159 | } 160 | decs[i].Sources.Exclude = v.Sources.Exclude 161 | decs[i].Sources.Include = v.Sources.Include 162 | for _, t := range []leeway.PackageType{leeway.DockerPackage, leeway.GenericPackage, leeway.GoPackage, leeway.YarnPackage} { 163 | vntcfg, ok := v.Config(t) 164 | if !ok { 165 | continue 166 | } 167 | decs[i].Config[t] = newConfigDescription(t, vntcfg) 168 | } 169 | } 170 | err = w.Write(decs) 171 | if err != nil { 172 | log.Fatal(err) 173 | } 174 | } 175 | }, 176 | } 177 | 178 | func init() { 179 | rootCmd.AddCommand(collectCmd) 180 | collectCmd.Flags().StringP("select", "l", "", "Filters packages by component constants (e.g. `-l foo` finds all packages whose components have a foo constant and `-l foo=bar` only prints packages whose components have a foo=bar constant)") 181 | 182 | addFormatFlags(collectCmd) 183 | } 184 | -------------------------------------------------------------------------------- /cmd/describe-const.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | log "github.com/sirupsen/logrus" 5 | "github.com/spf13/cobra" 6 | 7 | "github.com/gitpod-io/leeway/pkg/prettyprint" 8 | ) 9 | 10 | // describeConstCmd represents the describeTree command 11 | var describeConstCmd = &cobra.Command{ 12 | Use: "const", 13 | Short: "Prints the value of a component constant", 14 | Run: func(cmd *cobra.Command, args []string) { 15 | comp, pkg, _, exists := getTarget(args, false) 16 | if !exists { 17 | log.Fatal("const needs a component") 18 | } 19 | if comp == nil && pkg != nil { 20 | comp = pkg.C 21 | } 22 | 23 | type constDesc struct { 24 | Name string `json:"name" yaml:"name"` 25 | Value string `json:"value" yaml:"value"` 26 | } 27 | 28 | w := getWriterFromFlags(cmd) 29 | if w.Format == prettyprint.TemplateFormat && w.FormatString == "" { 30 | w.FormatString = `{{ range . }}{{ .Name }}:{{"\t"}}{{ .Value }}{{"\n"}}{{ end }}` 31 | } 32 | 33 | desc := make([]constDesc, 0, len(comp.Constants)) 34 | for k, v := range comp.Constants { 35 | desc = append(desc, constDesc{Name: k, Value: v}) 36 | } 37 | //nolint:errcheck 38 | w.Write(desc) 39 | }, 40 | } 41 | 42 | func init() { 43 | describeCmd.AddCommand(describeConstCmd) 44 | addFormatFlags(describeConstCmd) 45 | } 46 | -------------------------------------------------------------------------------- /cmd/describe-dependants.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | 9 | "github.com/gitpod-io/leeway/pkg/leeway" 10 | ) 11 | 12 | // describeDependantsCmd represents the describeDot command 13 | var describeDependantsCmd = &cobra.Command{ 14 | Use: "dependants", 15 | Short: "Describes the dependants package on the console", 16 | RunE: func(cmd *cobra.Command, args []string) error { 17 | var pkgs []*leeway.Package 18 | if len(args) > 0 { 19 | _, pkg, _, _ := getTarget(args, false) 20 | if pkg == nil { 21 | log.Fatal("graphview needs a package") 22 | } 23 | pkgs = []*leeway.Package{pkg} 24 | } else { 25 | ws, err := getWorkspace() 26 | if err != nil { 27 | log.Fatal(err) 28 | } 29 | 30 | allpkgs := ws.Packages 31 | for _, p := range allpkgs { 32 | for _, d := range p.Dependants() { 33 | delete(allpkgs, d.FullName()) 34 | } 35 | } 36 | for _, p := range allpkgs { 37 | pkgs = append(pkgs, p) 38 | } 39 | } 40 | 41 | transitive, _ := cmd.Flags().GetBool("transitive") 42 | for _, pkg := range pkgs { 43 | var deps []*leeway.Package 44 | if transitive { 45 | deps = pkg.TransitiveDependants() 46 | } else { 47 | deps = pkg.Dependants() 48 | } 49 | for _, d := range deps { 50 | fmt.Println(d.FullName()) 51 | } 52 | } 53 | 54 | return nil 55 | }, 56 | } 57 | 58 | func init() { 59 | describeCmd.AddCommand(describeDependantsCmd) 60 | describeDependantsCmd.Flags().BoolP("transitive", "t", false, "Print transitive dependants") 61 | } 62 | -------------------------------------------------------------------------------- /cmd/describe-dependencies.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/gookit/color" 7 | log "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | 10 | "github.com/gitpod-io/leeway/pkg/leeway" 11 | ) 12 | 13 | // describeDependenciesCmd represents the describeDot command 14 | var describeDependenciesCmd = &cobra.Command{ 15 | Use: "dependencies", 16 | Short: "Describes the depenencies package on the console, in Graphviz's dot format or as interactive website", 17 | RunE: func(cmd *cobra.Command, args []string) error { 18 | var pkgs []*leeway.Package 19 | if len(args) > 0 { 20 | _, pkg, _, _ := getTarget(args, false) 21 | if pkg == nil { 22 | log.Fatal("graphview needs a package") 23 | } 24 | pkgs = []*leeway.Package{pkg} 25 | } else { 26 | ws, err := getWorkspace() 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | 31 | allpkgs := ws.Packages 32 | for _, p := range allpkgs { 33 | for _, d := range p.GetDependencies() { 34 | delete(allpkgs, d.FullName()) 35 | } 36 | } 37 | for _, p := range allpkgs { 38 | pkgs = append(pkgs, p) 39 | } 40 | } 41 | 42 | if dot, _ := cmd.Flags().GetBool("dot"); dot { 43 | return printDepGraphAsDot(pkgs) 44 | } else { 45 | for _, pkg := range pkgs { 46 | printDepTree(pkg, 0) 47 | } 48 | } 49 | 50 | return nil 51 | }, 52 | } 53 | 54 | func printDepTree(pkg *leeway.Package, indent int) { 55 | var tpe string 56 | switch pkg.Type { 57 | case leeway.DockerPackage: 58 | tpe = "docker" 59 | case leeway.GenericPackage: 60 | tpe = "generic" 61 | case leeway.GoPackage: 62 | tpe = "go" 63 | case leeway.YarnPackage: 64 | tpe = "yarn" 65 | } 66 | 67 | fmt.Printf("%*s%s %s\n", indent, "", color.Gray.Sprintf("[%7s]", tpe), pkg.FullName()) 68 | for _, p := range pkg.GetDependencies() { 69 | printDepTree(p, indent+4) 70 | } 71 | } 72 | 73 | func printDepGraphAsDot(pkgs []*leeway.Package) error { 74 | var ( 75 | nodes = make(map[string]string) 76 | edges []string 77 | ) 78 | 79 | for _, pkg := range pkgs { 80 | allpkg := append(pkg.GetTransitiveDependencies(), pkg) 81 | for _, p := range allpkg { 82 | ver, err := p.Version() 83 | if err != nil { 84 | return err 85 | } 86 | if _, exists := nodes[ver]; exists { 87 | continue 88 | } 89 | nodes[ver] = fmt.Sprintf("p%s [label=\"%s\"];", ver, p.FullName()) 90 | } 91 | for _, p := range allpkg { 92 | ver, err := p.Version() 93 | if err != nil { 94 | return err 95 | } 96 | 97 | for _, dep := range p.GetDependencies() { 98 | depver, err := dep.Version() 99 | if err != nil { 100 | return err 101 | } 102 | edges = append(edges, fmt.Sprintf("p%s -> p%s;", ver, depver)) 103 | } 104 | } 105 | } 106 | 107 | fmt.Println("digraph G {") 108 | for _, n := range nodes { 109 | fmt.Printf(" %s\n", n) 110 | } 111 | for _, e := range edges { 112 | fmt.Printf(" %s\n", e) 113 | } 114 | fmt.Println("}") 115 | return nil 116 | } 117 | 118 | func init() { 119 | describeCmd.AddCommand(describeDependenciesCmd) 120 | 121 | describeDependenciesCmd.Flags().Bool("dot", false, "produce Graphviz dot output") 122 | } 123 | -------------------------------------------------------------------------------- /cmd/describe-environment-manifest.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | // describeEnvironmentManifestCmd represents the describeManifest command 11 | var describeEnvironmentManifestCmd = &cobra.Command{ 12 | Use: "environment-manifest", 13 | Short: "Prints the environment manifest of a workspace", 14 | Run: func(cmd *cobra.Command, args []string) { 15 | ws, err := getWorkspace() 16 | if err != nil { 17 | log.WithError(err).Fatal("cannot load workspace") 18 | } 19 | 20 | err = ws.EnvironmentManifest.Write(os.Stdout) 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | }, 25 | } 26 | 27 | func init() { 28 | describeCmd.AddCommand(describeEnvironmentManifestCmd) 29 | } 30 | -------------------------------------------------------------------------------- /cmd/describe-gitinfo.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/gitpod-io/leeway/pkg/leeway" 5 | log "github.com/sirupsen/logrus" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | // describeGitInfoCmd represents the describeTree command 10 | var describeGitInfoCmd = &cobra.Command{ 11 | Use: "git-info", 12 | Short: "Prints the Git info consumed by leeway", 13 | Run: func(cmd *cobra.Command, args []string) { 14 | comp, pkg, script, _ := getTarget(args, false) 15 | 16 | var ( 17 | nfo *leeway.GitInfo 18 | loc string 19 | ) 20 | if comp != nil { 21 | nfo = comp.Git() 22 | loc = comp.Origin 23 | } else if pkg != nil { 24 | nfo = pkg.C.Git() 25 | loc = pkg.C.Origin 26 | } else if script != nil { 27 | nfo = script.C.Git() 28 | loc = script.C.Origin 29 | } else { 30 | log.Fatal("no target given - try passing a package or component") 31 | } 32 | 33 | if nfo == nil { 34 | log.WithField("loc", loc).Fatal("not a Git working copy") 35 | } 36 | w := getWriterFromFlags(cmd) 37 | if w.FormatString == "" { 38 | w.FormatString = `dirty: {{.Dirty }} 39 | origin: {{ .Origin }} 40 | commit: {{ .Commit }} 41 | ` 42 | } 43 | err := w.Write(nfo) 44 | if err != nil { 45 | log.WithError(err).Fatal("cannot write git info") 46 | } 47 | }, 48 | } 49 | 50 | func init() { 51 | describeCmd.AddCommand(describeGitInfoCmd) 52 | addFormatFlags(describeGitInfoCmd) 53 | } 54 | -------------------------------------------------------------------------------- /cmd/describe-manifest.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | // describeManifestCmd represents the describeManifest command 11 | var describeManifestCmd = &cobra.Command{ 12 | Use: "manifest", 13 | Short: "Prints the version manifest (input for the version hash) of a package", 14 | Run: func(cmd *cobra.Command, args []string) { 15 | _, pkg, _, _ := getTarget(args, false) 16 | if pkg == nil { 17 | log.Fatal("manifest needs a package") 18 | } 19 | 20 | err := pkg.WriteVersionManifest(os.Stdout) 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | }, 25 | } 26 | 27 | func init() { 28 | describeCmd.AddCommand(describeManifestCmd) 29 | } 30 | -------------------------------------------------------------------------------- /cmd/describe-script.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | log "github.com/sirupsen/logrus" 5 | "github.com/spf13/cobra" 6 | 7 | "github.com/gitpod-io/leeway/pkg/prettyprint" 8 | ) 9 | 10 | // describeScriptCmd represents the describeTree command 11 | var describeScriptCmd = &cobra.Command{ 12 | Use: "script", 13 | Short: "Describes a script", 14 | Args: cobra.ExactArgs(1), 15 | Run: func(cmd *cobra.Command, args []string) { 16 | _, _, script, exists := getTarget(args, true) 17 | if !exists || script == nil { 18 | log.Fatal("needs a script") 19 | } 20 | 21 | w := getWriterFromFlags(cmd) 22 | if w.Format == prettyprint.TemplateFormat && w.FormatString == "" { 23 | w.FormatString = `Name:{{"\t"}}{{ .FullName }} 24 | {{ if .Description }}Description:{{"\t"}}{{ .Description }}{{ end }} 25 | Type:{{"\t"}}{{ .Type }} 26 | Workdir Layout:{{"\t"}}{{ .WorkdirLayout }} 27 | {{ if .Dependencies -}} 28 | Dependencies: 29 | {{- range $k, $v := .Dependencies }} 30 | {{"\t"}}{{ $v.FullName -}}{{"\t"}}{{ $v.Version -}} 31 | {{ end -}} 32 | {{ end }} 33 | ` 34 | } 35 | 36 | desc := newScriptDescription(script) 37 | err := w.Write(desc) 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | }, 42 | } 43 | 44 | func init() { 45 | describeCmd.AddCommand(describeScriptCmd) 46 | addFormatFlags(describeScriptCmd) 47 | } 48 | -------------------------------------------------------------------------------- /cmd/describe-tree.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/disiqueira/gotree" 7 | log "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | 10 | "github.com/gitpod-io/leeway/pkg/leeway" 11 | ) 12 | 13 | // describeTreeCmd represents the describeTree command 14 | var describeTreeCmd = &cobra.Command{ 15 | Use: "tree", 16 | Short: "Prints the depepency tree of a package", 17 | Run: func(cmd *cobra.Command, args []string) { 18 | _, pkg, _, _ := getTarget(args, false) 19 | if pkg == nil { 20 | log.Fatal("tree needs a package") 21 | } 22 | 23 | var print func(parent gotree.Tree, pkg *leeway.Package) 24 | print = func(parent gotree.Tree, pkg *leeway.Package) { 25 | n := parent.Add(pkg.FullName()) 26 | for _, dep := range pkg.GetDependencies() { 27 | print(n, dep) 28 | } 29 | } 30 | 31 | tree := gotree.New("WORKSPACE") 32 | print(tree, pkg) 33 | _, err := fmt.Println(tree.Print()) 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | }, 38 | } 39 | 40 | func init() { 41 | describeCmd.AddCommand(describeTreeCmd) 42 | } 43 | -------------------------------------------------------------------------------- /cmd/experiemental-unmount.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package cmd 5 | 6 | import ( 7 | "fmt" 8 | "io" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "syscall" 13 | 14 | "github.com/sirupsen/logrus" 15 | "github.com/spf13/cobra" 16 | ) 17 | 18 | // unmountCmd represents the version command 19 | var unmountCmd = &cobra.Command{ 20 | Use: "unmount ", 21 | Short: "[experimental] Unmounts a previously mounted overlay", 22 | Args: cobra.ExactArgs(1), 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | mp := args[0] 25 | origin, upper, delmp, err := findOverlayMount(mp) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | err = syscall.Unmount(mp, 0) 31 | if err != nil { 32 | return err 33 | } 34 | defer func() { 35 | err = syscall.Unmount(delmp, 0) 36 | if err != nil { 37 | logrus.WithError(err).WithField("mountpoint", delmp).Error("cannot unmount delup overlay") 38 | } 39 | }() 40 | 41 | applyChanges, _ := cmd.Flags().GetBool("apply") 42 | if !applyChanges { 43 | return nil 44 | } 45 | 46 | err = filepath.Walk(upper, func(path string, info os.FileInfo, err error) error { 47 | if err != nil { 48 | return err 49 | } 50 | 51 | dst := filepath.Join(origin, strings.TrimPrefix(path, upper)) 52 | if path == upper || dst == origin { 53 | return nil 54 | } 55 | 56 | if info.Mode()&os.ModeCharDevice == os.ModeCharDevice { 57 | logrus.WithField("dest", dst).Debug("applying change: deleting file") 58 | err = os.RemoveAll(dst) 59 | if err != nil && !os.IsNotExist(err) { 60 | return err 61 | } 62 | return nil 63 | } 64 | 65 | if info.IsDir() { 66 | logrus.WithField("dest", dst).Debug("applying change: creating directory") 67 | err = os.MkdirAll(dst, info.Mode()) 68 | if err != nil && !os.IsExist(err) { 69 | return err 70 | } 71 | stat := info.Sys().(*syscall.Stat_t) 72 | err = os.Chown(dst, int(stat.Uid), int(stat.Gid)) 73 | if err != nil { 74 | return err 75 | } 76 | return nil 77 | } 78 | 79 | src, err := os.Open(path) 80 | if err != nil { 81 | return err 82 | } 83 | defer src.Close() 84 | 85 | f, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) 86 | if err != nil { 87 | return err 88 | } 89 | defer f.Close() 90 | 91 | logrus.WithField("dest", dst).Debug("applying change: copying content") 92 | _, err = io.Copy(f, src) 93 | if err != nil { 94 | return err 95 | } 96 | 97 | stat := info.Sys().(*syscall.Stat_t) 98 | err = os.Chown(dst, int(stat.Uid), int(stat.Gid)) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | return nil 104 | }) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | return nil 110 | }, 111 | } 112 | 113 | func findOverlayMount(mountpoint string) (origin, upper, delmp string, err error) { 114 | mnts, err := os.ReadFile("/proc/mounts") 115 | if err != nil { 116 | return 117 | } 118 | 119 | for _, row := range strings.Split(string(mnts), "\n") { 120 | segs := strings.Split(row, " ") 121 | if len(segs) < 4 { 122 | continue 123 | } 124 | if segs[1] != mountpoint { 125 | continue 126 | } 127 | 128 | opts := strings.Split(segs[3], ",") 129 | for _, opt := range opts { 130 | if strings.HasPrefix(opt, "lowerdir=") { 131 | delmp = strings.TrimPrefix(opt, "lowerdir=") 132 | } else if strings.HasPrefix(opt, "upperdir=") { 133 | upper = strings.TrimPrefix(opt, "upperdir=") 134 | } 135 | } 136 | if delmp == "" { 137 | err = fmt.Errorf("did not find lowerdir") 138 | return 139 | } 140 | if upper == "" { 141 | err = fmt.Errorf("did not find upperdir") 142 | return 143 | } 144 | } 145 | for _, row := range strings.Split(string(mnts), "\n") { 146 | segs := strings.Split(row, " ") 147 | if len(segs) < 4 { 148 | continue 149 | } 150 | if segs[1] != delmp { 151 | continue 152 | } 153 | 154 | opts := strings.Split(segs[3], ",") 155 | for _, opt := range opts { 156 | if strings.HasPrefix(opt, "lowerdir=") { 157 | origin = strings.TrimPrefix(opt, "lowerdir=") 158 | } 159 | } 160 | if origin == "" { 161 | err = fmt.Errorf("did not find origin") 162 | return 163 | } 164 | } 165 | 166 | if origin == "" { 167 | err = fmt.Errorf("did not find mountpoint") 168 | return 169 | } 170 | 171 | return 172 | } 173 | 174 | func init() { 175 | addExperimentalCommand(rootCmd, unmountCmd) 176 | 177 | unmountCmd.Flags().Bool("apply", true, "apply the changes made in the overlay back to the original workspace") 178 | } 179 | -------------------------------------------------------------------------------- /cmd/experimental-mount.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package cmd 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | "path/filepath" 10 | "syscall" 11 | 12 | "github.com/spf13/cobra" 13 | 14 | "github.com/gitpod-io/leeway/pkg/leeway" 15 | ) 16 | 17 | // mountCmd represents the mount command 18 | var mountCmd = &cobra.Command{ 19 | Use: "mount ", 20 | Short: "[experimental] Mounts a package or workspace variant", 21 | Args: cobra.MinimumNArgs(1), 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | ws, err := getWorkspace() 24 | if err != nil { 25 | return fmt.Errorf("cannot load workspace: %q", err) 26 | } 27 | 28 | dest := args[0] 29 | err = os.MkdirAll(dest, 0777) 30 | if err != nil && !os.IsExist(err) { 31 | return fmt.Errorf("cannot create destination dir: %q", err) 32 | } 33 | 34 | wdbase, _ := cmd.Flags().GetString("workdir") 35 | if wdbase != "" { 36 | err = os.MkdirAll(wdbase, 0777) 37 | } else { 38 | wdbase, err = os.MkdirTemp(filepath.Dir(dest), "leeway-workdir-*") 39 | } 40 | if err != nil && !os.IsExist(err) { 41 | return err 42 | } 43 | var ( 44 | delup = filepath.Join(wdbase, "delup") 45 | delmp = filepath.Join(wdbase, "delmp") 46 | wd = filepath.Join(wdbase, "work") 47 | upper = filepath.Join(wdbase, "upper") 48 | ) 49 | for _, p := range []string{delup, delmp, wd, upper} { 50 | err = os.MkdirAll(p, 0777) 51 | if err != nil && !os.IsExist(err) { 52 | return err 53 | } 54 | } 55 | 56 | // prepare delup 57 | err = syscall.Mount("overlay", delmp, "overlay", 0, fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", ws.Origin, delup, wd)) 58 | if err != nil { 59 | return fmt.Errorf("cannot mount delup overlay: %q", err) 60 | } 61 | strict, _ := cmd.Flags().GetBool("strict") 62 | err = leeway.DeleteNonWorkspaceFiles(delmp, &ws, strict) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | // actually mount overlay 68 | err = syscall.Mount("overlay", dest, "overlay", 0, fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", delmp, upper, wd)) 69 | if err != nil { 70 | return fmt.Errorf("cannot mount overlay: %q", err) 71 | } 72 | 73 | return nil 74 | }, 75 | } 76 | 77 | func init() { 78 | addExperimentalCommand(rootCmd, mountCmd) 79 | 80 | mountCmd.Flags().String("workdir", "", "overlayfs workdir location (must be on the same volume as the destination)") 81 | mountCmd.Flags().Bool("strict", false, "keep only package source files") 82 | } 83 | -------------------------------------------------------------------------------- /cmd/export.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | 9 | "github.com/gitpod-io/leeway/pkg/leeway" 10 | ) 11 | 12 | // exportCmd represents the version command 13 | var exportCmd = &cobra.Command{ 14 | Use: "export ", 15 | Short: "Copies a workspace to the destination", 16 | Args: cobra.ExactArgs(1), 17 | RunE: func(cmd *cobra.Command, args []string) error { 18 | if _, err := os.Stat(args[0]); err == nil { 19 | return fmt.Errorf("destination %s exists already", args[0]) 20 | } 21 | 22 | workspace, err := getWorkspace() 23 | if err != nil { 24 | return err 25 | } 26 | 27 | strict, _ := cmd.Flags().GetBool("strict") 28 | return leeway.CopyWorkspace(args[0], &workspace, strict) 29 | }, 30 | } 31 | 32 | func init() { 33 | rootCmd.AddCommand(exportCmd) 34 | 35 | exportCmd.Flags().Bool("strict", false, "keep only package source files") 36 | } 37 | -------------------------------------------------------------------------------- /cmd/fmt.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/spf13/cobra" 11 | 12 | "github.com/gitpod-io/leeway/pkg/leeway" 13 | ) 14 | 15 | // fmtCmd represents the version command 16 | var fmtCmd = &cobra.Command{ 17 | Use: "fmt [files...]", 18 | Short: "Formats BUILD.yaml files", 19 | RunE: func(cmd *cobra.Command, args []string) error { 20 | fns := args 21 | if len(fns) == 0 { 22 | ws, err := getWorkspace() 23 | if err != nil { 24 | return err 25 | } 26 | for _, comp := range ws.Components { 27 | fns = append(fns, filepath.Join(comp.Origin, "BUILD.yaml")) 28 | } 29 | } 30 | 31 | var ( 32 | inPlace, _ = cmd.Flags().GetBool("in-place") 33 | fix, _ = cmd.Flags().GetBool("fix") 34 | ) 35 | for _, fn := range fns { 36 | err := formatBuildYaml(fn, inPlace, fix) 37 | if err != nil { 38 | return err 39 | } 40 | } 41 | 42 | return nil 43 | }, 44 | } 45 | 46 | func formatBuildYaml(fn string, inPlace, fix bool) error { 47 | f, err := os.OpenFile(fn, os.O_RDWR, 0644) 48 | if err != nil { 49 | return err 50 | } 51 | defer f.Close() 52 | 53 | var out io.Writer = os.Stdout 54 | if inPlace { 55 | buf := bytes.NewBuffer(nil) 56 | out = buf 57 | //nolint:errcheck 58 | defer func() { 59 | f.Seek(0, 0) 60 | f.Truncate(0) 61 | 62 | io.Copy(f, buf) 63 | }() 64 | } else { 65 | fmt.Printf("---\n# %s\n", fn) 66 | } 67 | 68 | err = leeway.FormatBUILDyaml(out, f, fix) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | return nil 74 | } 75 | 76 | func init() { 77 | rootCmd.AddCommand(fmtCmd) 78 | 79 | fmtCmd.Flags().BoolP("in-place", "i", false, "format file in place rather than printing it to stdout") 80 | fmtCmd.Flags().BoolP("fix", "f", false, "fix issues other than formatting (e.g. deprecated package types)") 81 | } 82 | -------------------------------------------------------------------------------- /cmd/init.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "strings" 8 | 9 | log "github.com/sirupsen/logrus" 10 | "github.com/spf13/cobra" 11 | "gopkg.in/yaml.v3" 12 | 13 | "github.com/gitpod-io/leeway/pkg/leeway" 14 | ) 15 | 16 | var ( 17 | dockerfileCandidates = []string{"Dockerfile", "leeway.Dockerfile"} 18 | packageTypeDetectionFiles = map[leeway.PackageType][]string{ 19 | leeway.DockerPackage: dockerfileCandidates, 20 | leeway.GoPackage: {"go.mod", "go.sum"}, 21 | leeway.YarnPackage: {"package.json", "yarn.lock"}, 22 | } 23 | initPackageGenerator = map[leeway.PackageType]func(name string) ([]byte, error){ 24 | leeway.DockerPackage: initDockerPackage, 25 | leeway.GoPackage: initGoPackage, 26 | leeway.YarnPackage: initYarnPackage, 27 | leeway.GenericPackage: initGenericPackage, 28 | } 29 | ) 30 | 31 | // initCmd represents the version command 32 | var initCmd = &cobra.Command{ 33 | Use: "init ", 34 | Short: "Initializes a new leeway package (and component if need be) in the current directory", 35 | Args: cobra.ExactArgs(1), 36 | ValidArgs: []string{"go", "yarn", "docker", "generic"}, 37 | RunE: func(cmd *cobra.Command, args []string) error { 38 | var tpe leeway.PackageType 39 | if tper, _ := cmd.Flags().GetString("type"); tper != "" { 40 | tpe = leeway.PackageType(tper) 41 | } else { 42 | tpe = detectPossiblePackageType() 43 | } 44 | 45 | generator, ok := initPackageGenerator[tpe] 46 | if !ok { 47 | return fmt.Errorf("unknown package type: %q", tpe) 48 | } 49 | 50 | tpl, err := generator(args[0]) 51 | if err != nil { 52 | return err 53 | } 54 | var pkg yaml.Node 55 | err = yaml.Unmarshal(tpl, &pkg) 56 | if err != nil { 57 | log.WithField("template", string(tpl)).Warn("broken package template") 58 | return fmt.Errorf("This is a leeway bug. Cannot parse package template: %w", err) 59 | } 60 | 61 | f, err := os.OpenFile("BUILD.yaml", os.O_CREATE|os.O_RDWR, 0644) 62 | if err != nil { 63 | return err 64 | } 65 | defer f.Close() 66 | 67 | var cmp yaml.Node 68 | err = yaml.NewDecoder(f).Decode(&cmp) 69 | if err == io.EOF { 70 | err = yaml.Unmarshal([]byte(`packages: []`), &cmp) 71 | } 72 | if err != nil { 73 | return err 74 | } 75 | 76 | cmps := cmp.Content[0].Content 77 | for i, nde := range cmps { 78 | if !(nde.Value == "packages" && i < len(cmps)-1 && cmps[i+1].Kind == yaml.SequenceNode) { 79 | continue 80 | } 81 | 82 | pkgs := cmps[i+1] 83 | pkgs.Style = yaml.FoldedStyle 84 | pkgs.Content = append(pkgs.Content, pkg.Content[0]) 85 | cmps[i+1] = pkgs 86 | } 87 | cmp.Content[0].Content = cmps 88 | 89 | _, err = f.Seek(0, 0) 90 | if err != nil { 91 | return err 92 | } 93 | err = yaml.NewEncoder(f).Encode(&cmp) 94 | if err != nil { 95 | return err 96 | } 97 | 98 | return nil 99 | }, 100 | } 101 | 102 | func detectPossiblePackageType() leeway.PackageType { 103 | for tpe, fns := range packageTypeDetectionFiles { 104 | for _, fn := range fns { 105 | _, err := os.Stat(fn) 106 | if err != nil { 107 | continue 108 | } 109 | 110 | return tpe 111 | } 112 | } 113 | 114 | return leeway.GenericPackage 115 | } 116 | 117 | func initGoPackage(name string) ([]byte, error) { 118 | return []byte(fmt.Sprintf(`name: %s 119 | type: go 120 | srcs: 121 | - go.mod 122 | - go.sum 123 | - "**/*.go" 124 | env: 125 | - CGO_ENABLED=0 126 | config: 127 | packaging: app 128 | `, name)), nil 129 | } 130 | 131 | func initDockerPackage(name string) ([]byte, error) { 132 | var dockerfile string 133 | for _, f := range dockerfileCandidates { 134 | if _, err := os.Stat(f); err == nil { 135 | dockerfile = f 136 | break 137 | } 138 | } 139 | if dockerfile == "" { 140 | return nil, fmt.Errorf("no Dockerfile found") 141 | } 142 | 143 | log.Warnf("Please update your BUILD.yaml and change the image reference of the new \"%s\" package", name) 144 | return []byte(fmt.Sprintf(`name: %s 145 | type: docker 146 | config: 147 | dockerfile: %s 148 | image: some/imgage/in/some:repo`, name, dockerfile)), nil 149 | } 150 | 151 | func initYarnPackage(name string) ([]byte, error) { 152 | return []byte(fmt.Sprintf(`name: %s 153 | type: yarn 154 | srcs: 155 | - package.json 156 | - "src/**" 157 | config: 158 | yarnLock: yarn.lock 159 | tsconfig: tsconfig.json 160 | `, name)), nil 161 | } 162 | 163 | func initGenericPackage(name string) ([]byte, error) { 164 | fs, err := os.ReadDir(".") 165 | if err != nil { 166 | return nil, err 167 | } 168 | var srcs []string 169 | for _, f := range fs { 170 | if f.Name() == "BUILD.yaml" { 171 | continue 172 | } 173 | if strings.HasPrefix(f.Name(), ".") { 174 | continue 175 | } 176 | 177 | var da string 178 | if f.IsDir() { 179 | da = "/**" 180 | } 181 | srcs = append(srcs, fmt.Sprintf(" - \"%s%s\"", f.Name(), da)) 182 | } 183 | 184 | log.Warnf("Please update your BUILD.yaml and change the commands of the new \"%s\" package", name) 185 | return []byte(fmt.Sprintf(`name: %s 186 | type: generic 187 | srcs: 188 | %s 189 | config: 190 | comamnds: 191 | - ["echo", "commands", "go", "here"] 192 | `, name, strings.Join(srcs, "\n"))), nil 193 | } 194 | 195 | func init() { 196 | rootCmd.AddCommand(initCmd) 197 | 198 | initCmd.Flags().StringP("type", "t", "", "type of the new package") 199 | } 200 | -------------------------------------------------------------------------------- /cmd/link.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | 10 | "github.com/gitpod-io/leeway/pkg/linker" 11 | ) 12 | 13 | // linkCmd represents the version command 14 | var linkCmd = &cobra.Command{ 15 | Use: "link", 16 | Short: "Links all packages in-situ", 17 | RunE: func(cmd *cobra.Command, args []string) error { 18 | ws, err := getWorkspace() 19 | if err != nil { 20 | return err 21 | } 22 | _, pkg, _, _ := getTarget(args, false) 23 | 24 | switch val, _ := cmd.Flags().GetString("go-link"); val { 25 | case "auto": 26 | if _, ferr := os.Stat(filepath.Join(ws.Origin, "go.work")); ferr == nil { 27 | err = linker.LinkGoWorkspace(&ws) 28 | } else { 29 | err = linker.LinkGoModules(&ws, pkg) 30 | } 31 | case "module": 32 | err = linker.LinkGoModules(&ws, pkg) 33 | case "workspace": 34 | err = linker.LinkGoWorkspace(&ws) 35 | } 36 | if err != nil { 37 | return err 38 | } 39 | 40 | if ok, _ := cmd.Flags().GetBool("yarn2-link"); ok { 41 | err = linker.LinkYarnPackagesWithYarn2(&ws) 42 | if err != nil { 43 | return err 44 | } 45 | } else { 46 | log.Info("yarn2 package linking disabled") 47 | } 48 | 49 | return nil 50 | }, 51 | } 52 | 53 | func init() { 54 | rootCmd.AddCommand(linkCmd) 55 | 56 | linkCmd.Flags().Bool("yarn2-link", false, "link yarn packages using yarn2 resolutions") 57 | linkCmd.Flags().String("go-link", "auto", "link Go modules or workspace. Valid values are auto, module or workspace") 58 | } 59 | -------------------------------------------------------------------------------- /cmd/plumbing-exec.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "os" 7 | "os/exec" 8 | "strings" 9 | 10 | log "github.com/sirupsen/logrus" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | // plumbingExecCmd represents the version command 15 | var plumbingExecCmd = &cobra.Command{ 16 | Use: "exec ", 17 | Short: "Executes commands", 18 | Args: cobra.ExactArgs(1), 19 | Run: func(cmd *cobra.Command, args []string) { 20 | input, err := os.ReadFile(args[0]) 21 | if err != nil { 22 | log.WithError(err).Fatal("cannot read input file") 23 | } 24 | 25 | dec, err := base64.StdEncoding.DecodeString(string(input)) 26 | if err != nil { 27 | log.WithError(err).Fatal("failed to base64 decode commands") 28 | } 29 | 30 | var commands [][]string 31 | err = json.Unmarshal(dec, &commands) 32 | if err != nil { 33 | log.WithError(err).Fatal("failed to unmarshal commands") 34 | } 35 | 36 | for _, c := range commands { 37 | name, args := c[0], c[1:] 38 | log.WithField("command", strings.Join(append([]string{name}, args...), " ")).Debug("running") 39 | 40 | cmd := exec.Command(name, args...) 41 | cmd.Stdout = os.Stdout 42 | cmd.Stderr = os.Stderr 43 | cmd.Stdin = os.Stdin 44 | err := cmd.Run() 45 | if err != nil { 46 | os.Exit(1) 47 | } 48 | } 49 | }, 50 | } 51 | 52 | func init() { 53 | plumbingCmd.AddCommand(plumbingExecCmd) 54 | } 55 | -------------------------------------------------------------------------------- /cmd/plumbing.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | // plumbingCmd represents the version command 8 | var plumbingCmd = &cobra.Command{ 9 | Use: "plumbing", 10 | Short: "Internal commands used by leeway itself", 11 | Hidden: true, 12 | Args: cobra.MinimumNArgs(1), 13 | } 14 | 15 | func init() { 16 | rootCmd.AddCommand(plumbingCmd) 17 | } 18 | -------------------------------------------------------------------------------- /cmd/provenance-assert.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "io" 7 | "os" 8 | "strings" 9 | 10 | "github.com/gitpod-io/leeway/pkg/leeway" 11 | "github.com/gitpod-io/leeway/pkg/provutil" 12 | "github.com/in-toto/in-toto-golang/in_toto" 13 | log "github.com/sirupsen/logrus" 14 | "github.com/spf13/cobra" 15 | "sigs.k8s.io/bom/pkg/provenance" 16 | ) 17 | 18 | // provenanceExportCmd represents the provenance assert command 19 | var provenanceAssertCmd = &cobra.Command{ 20 | Use: "assert ", 21 | Short: "Makes assertions about the provenance of a package", 22 | Args: cobra.ExactArgs(1), 23 | Run: func(cmd *cobra.Command, args []string) { 24 | bundleFN, pkgFN, pkg, err := getProvenanceTarget(cmd, args) 25 | if err != nil { 26 | log.WithError(err).Fatal("cannot locate bundle") 27 | } 28 | 29 | var assertions provutil.Assertions 30 | if signed, err := cmd.Flags().GetBool("signed"); err != nil { 31 | log.Fatal(err) 32 | } else if signed { 33 | log.Warn("checking signatures is most likely broken and will probably return false results") 34 | 35 | var keyPath string 36 | if pkg == nil { 37 | keyPath = os.Getenv("LEEWAY_PROVENANCE_KEYPATH") 38 | } else { 39 | keyPath = pkg.C.W.Provenance.KeyPath 40 | } 41 | if keyPath == "" { 42 | log.Fatal("no key path specified - use the LEEWAY_PROVENANCE_KEYPATH to specify one") 43 | } 44 | 45 | var key in_toto.Key 46 | err := key.LoadKeyDefaults(keyPath) 47 | if err != nil { 48 | log.WithError(err).Fatal("cannot load key from " + pkg.C.W.Provenance.KeyPath) 49 | } 50 | assertions = append(assertions, provutil.AssertSignedWith(key)) 51 | } 52 | if do, err := cmd.Flags().GetBool("built-with-leeway"); err != nil { 53 | log.Fatal(err) 54 | } else if do { 55 | assertions = append(assertions, provutil.AssertBuiltWithLeeway) 56 | } 57 | if ver, err := cmd.Flags().GetString("built-with-leeway-version"); err != nil { 58 | log.Fatal(err) 59 | } else if ver != "" { 60 | assertions = append(assertions, provutil.AssertBuiltWithLeewayVersion(ver)) 61 | } 62 | if do, err := cmd.Flags().GetBool("git-only"); err != nil { 63 | log.Fatal(err) 64 | } else if do { 65 | assertions = append(assertions, provutil.AssertGitMaterialOnly) 66 | } 67 | 68 | var failures []provutil.Violation 69 | stmt := provenance.NewSLSAStatement() 70 | assert := func(env *provenance.Envelope) error { 71 | if env.PayloadType != in_toto.PayloadType { 72 | log.Warnf("only supporting %s payloads, not %s - skipping", in_toto.PayloadType, env.PayloadType) 73 | return nil 74 | } 75 | 76 | failures = append(assertions.AssertBundle(env), failures...) 77 | 78 | raw, err := base64.StdEncoding.DecodeString(env.Payload) 79 | if err != nil { 80 | return err 81 | } 82 | err = json.Unmarshal(raw, &stmt) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | failures = append(assertions.AssertStatement(stmt), failures...) 88 | 89 | return nil 90 | } 91 | 92 | if pkg == nil { 93 | var f *os.File 94 | f, err = os.Open(bundleFN) 95 | if err != nil { 96 | log.WithError(err).Fatalf("cannot open attestation bundle %s", bundleFN) 97 | } 98 | defer f.Close() 99 | 100 | err = provutil.DecodeBundle(f, assert) 101 | } else { 102 | err = leeway.AccessAttestationBundleInCachedArchive(pkgFN, func(bundle io.Reader) error { 103 | return provutil.DecodeBundle(bundle, assert) 104 | }) 105 | } 106 | if err != nil { 107 | log.WithError(err).Fatal("cannot assert attestation bundle") 108 | } 109 | 110 | if len(failures) != 0 { 111 | for _, f := range failures { 112 | log.Error(f.String()) 113 | } 114 | log.Fatal("failed") 115 | } 116 | }, 117 | } 118 | 119 | func getProvenanceTarget(cmd *cobra.Command, args []string) (bundleFN, pkgFN string, pkg *leeway.Package, err error) { 120 | if strings.HasPrefix(args[0], "file://") { 121 | bundleFN = strings.TrimPrefix(args[0], "file://") 122 | } else { 123 | _, pkg, _, _ = getTarget(args, false) 124 | if pkg == nil { 125 | log.Fatal("provenance export requires a package") 126 | } 127 | 128 | _, cache := getBuildOpts(cmd) 129 | 130 | var ok bool 131 | pkgFN, ok = cache.Location(pkg) 132 | if !ok { 133 | log.Fatalf("%s is not built", pkg.FullName()) 134 | } 135 | } 136 | return 137 | } 138 | 139 | func init() { 140 | provenanceAssertCmd.Flags().Bool("signed", false, "ensure that all entries in the attestation bundle are signed and valid under the given key") 141 | provenanceAssertCmd.Flags().Bool("built-with-leeway", false, "ensure that all entries in the attestation bundle are built by leeway") 142 | provenanceAssertCmd.Flags().String("built-with-leeway-version", "", "ensure that all entries in the attestation bundle are built by a specific leeway version") 143 | provenanceAssertCmd.Flags().Bool("git-only", false, "ensure that all entries in the attestation bundle are built directly from Git (i.e. only have git material entries)") 144 | 145 | addBuildFlags(provenanceAssertCmd) 146 | provenanceCmd.AddCommand(provenanceAssertCmd) 147 | } 148 | -------------------------------------------------------------------------------- /cmd/provenance-export.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "io" 7 | "os" 8 | 9 | "github.com/gitpod-io/leeway/pkg/leeway" 10 | "github.com/gitpod-io/leeway/pkg/provutil" 11 | log "github.com/sirupsen/logrus" 12 | "github.com/spf13/cobra" 13 | "sigs.k8s.io/bom/pkg/provenance" 14 | ) 15 | 16 | // provenanceExportCmd represents the provenance export command 17 | var provenanceExportCmd = &cobra.Command{ 18 | Use: "export ", 19 | Short: "Exports the provenance bundle of a (previously built) package", 20 | Args: cobra.MinimumNArgs(1), 21 | Run: func(cmd *cobra.Command, args []string) { 22 | bundleFN, pkgFN, pkg, err := getProvenanceTarget(cmd, args) 23 | if err != nil { 24 | log.WithError(err).Fatal("cannot locate bundle") 25 | } 26 | 27 | decode, _ := cmd.Flags().GetBool("decode") 28 | out := json.NewEncoder(os.Stdout) 29 | 30 | export := func(env *provenance.Envelope) error { 31 | if !decode { 32 | return out.Encode(env) 33 | } 34 | 35 | dec, err := base64.StdEncoding.DecodeString(env.Payload) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | // we make a Marshal(Unmarshal(...)) detour here to ensure we're still outputing 41 | // newline delimited JSON. We have no idea how the payload actually looks like, just 42 | // that it's valid JSON. 43 | var decc map[string]interface{} 44 | err = json.Unmarshal(dec, &decc) 45 | if err != nil { 46 | return err 47 | } 48 | err = out.Encode(decc) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | return nil 54 | } 55 | 56 | if pkg == nil { 57 | f, err := os.Open(bundleFN) 58 | if err != nil { 59 | log.WithError(err).Fatal("cannot open attestation bundle") 60 | } 61 | defer f.Close() 62 | err = provutil.DecodeBundle(f, export) 63 | if err != nil { 64 | log.WithError(err).Fatal("cannot extract attestation bundle") 65 | } 66 | } else { 67 | err = leeway.AccessAttestationBundleInCachedArchive(pkgFN, func(bundle io.Reader) error { 68 | return provutil.DecodeBundle(bundle, export) 69 | }) 70 | if err != nil { 71 | log.WithError(err).Fatal("cannot extract attestation bundle") 72 | } 73 | } 74 | 75 | }, 76 | } 77 | 78 | func init() { 79 | provenanceExportCmd.Flags().Bool("decode", false, "decode the base64 payload of the envelopes") 80 | 81 | provenanceCmd.AddCommand(provenanceExportCmd) 82 | addBuildFlags(provenanceExportCmd) 83 | } 84 | -------------------------------------------------------------------------------- /cmd/provenance.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | // provenanceCmd represents the provenance command 8 | var provenanceCmd = &cobra.Command{ 9 | Use: "provenance ", 10 | Short: "Helpful commands for inspecing package provenance", 11 | Args: cobra.MinimumNArgs(1), 12 | Aliases: []string{"prov"}, 13 | } 14 | 15 | func init() { 16 | rootCmd.AddCommand(provenanceCmd) 17 | } 18 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "runtime/trace" 8 | "strings" 9 | 10 | "github.com/gookit/color" 11 | log "github.com/sirupsen/logrus" 12 | "github.com/spf13/cobra" 13 | "golang.org/x/xerrors" 14 | 15 | "github.com/gitpod-io/leeway/pkg/leeway" 16 | ) 17 | 18 | const ( 19 | // EnvvarWorkspaceRoot names the environment variable we check for the workspace root path 20 | EnvvarWorkspaceRoot = "LEEWAY_WORKSPACE_ROOT" 21 | 22 | // EnvvarRemoteCacheBucket configures a bucket name. This enables the use of RemoteStorage 23 | EnvvarRemoteCacheBucket = "LEEWAY_REMOTE_CACHE_BUCKET" 24 | 25 | // EnvvarRemoteCacheStorage configures a Remote Storage Provider. Default is GCP 26 | EnvvarRemoteCacheStorage = "LEEWAY_REMOTE_CACHE_STORAGE" 27 | ) 28 | 29 | const ( 30 | bashCompletionFunc = `__leeway_parse_get() 31 | { 32 | local leeway_output out 33 | if leeway_output=$(leeway collect 2>/dev/null); then 34 | out=($(echo "${leeway_output}" | awk '{print $1}')) 35 | COMPREPLY=( $( compgen -W "${out[*]}" -- "$cur" ) ) 36 | fi 37 | } 38 | 39 | __leeway_get_resource() 40 | { 41 | __leeway_parse_get 42 | if [[ $? -eq 0 ]]; then 43 | return 0 44 | fi 45 | } 46 | 47 | __leeway_custom_func() { 48 | case ${last_command} in 49 | leeway_build | leeway_describe) 50 | __leeway_get_resource 51 | return 52 | ;; 53 | *) 54 | ;; 55 | esac 56 | } 57 | ` 58 | ) 59 | 60 | var ( 61 | workspace string 62 | buildArgs []string 63 | verbose bool 64 | variant string 65 | ) 66 | 67 | // rootCmd represents the base command when called without any subcommands 68 | var rootCmd = &cobra.Command{ 69 | Use: "leeway", 70 | Short: "A caching meta-build system", 71 | Long: color.Render(`Leeway is a heavily caching build system for Go, Yarn and Docker projects. It knows three core concepts: 72 | Workspace: the workspace is the root of all operations. All component names are relative to this path. No relevant 73 | file must be placed outside the workspace. The workspace root is marked with a WORKSPACE file. 74 | Component: a component is single piece of standalone software. Every folder in the workspace which contains a BUILD file 75 | is a component. Components are identifed by their path relative to the workspace root. 76 | Package: packages are the buildable unit in leeway. Every component can define multiple packages in its build file. 77 | Packages are identified by their name prefixed with the component name, e.g. some-component:pkg 78 | 79 | Configuration 80 | Leeway is configured exclusively through the WORKSPACE/BUILD files and environment variables. The following environment 81 | variables have an effect on leeway: 82 | LEEWAY_WORKSPACE_ROOT Contains the path where to look for a WORKSPACE file. Can also be set using --workspace. 83 | LEEWAY_REMOTE_CACHE_STORAGE Defines the remote caching storage provider. Valid values are "GCP" and "AWS". Defaults to "GCP". 84 | LEEWAY_REMOTE_CACHE_BUCKET Enables remote caching using GCP or S3 buckets. Required credentials depend on the storage provider: 85 | - GCP: leeway expects "gsutil" in the path configured and authenticated so that it can work with the bucket. 86 | - AWS: leeway expects that AWS credentials have been provided and with read/write access to the S3 bucket. 87 | For details on configuring AWS credentials see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html 88 | LEEWAY_CACHE_DIR Location of the local build cache. The directory does not have to exist yet. 89 | LEEWAY_BUILD_DIR Working location of leeway (i.e. where the actual builds happen). This location will see heavy I/O 90 | which makes it advisable to place this on a fast SSD or in RAM. 91 | LEEWAY_YARN_MUTEX Configures the mutex flag leeway will pass to yarn. Defaults to "network". 92 | See https://yarnpkg.com/lang/en/docs/cli/#toc-concurrency-and-mutex for possible values. 93 | LEEWAY_DEFAULT_CACHE_LEVEL Sets the default cache level for builds. Defaults to "remote". 94 | LEEWAY_EXPERIMENTAL Enables experimental leeway features and commands. 95 | `), 96 | PersistentPreRun: func(cmd *cobra.Command, args []string) { 97 | if verbose { 98 | log.SetLevel(log.DebugLevel) 99 | } 100 | }, 101 | BashCompletionFunction: bashCompletionFunc, 102 | } 103 | 104 | // Execute adds all child commands to the root command and sets flags appropriately. 105 | // This is called by main.main(). It only needs to happen once to the rootCmd. 106 | func Execute() { 107 | tp := os.Getenv("LEEWAY_TRACE") 108 | if tp != "" { 109 | f, err := os.OpenFile(tp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 110 | if err != nil { 111 | log.WithError(err).Fatal("cannot start trace but LEEWAY_TRACE is set") 112 | return 113 | } 114 | defer f.Close() 115 | err = trace.Start(f) 116 | if err != nil { 117 | log.WithError(err).Fatal("cannot start trace but LEEWAY_TRACE is set") 118 | return 119 | } 120 | defer trace.Stop() 121 | 122 | defer trace.StartRegion(context.Background(), "main").End() 123 | } 124 | 125 | if err := rootCmd.Execute(); err != nil { 126 | fmt.Println(err) 127 | os.Exit(1) 128 | } 129 | } 130 | 131 | func init() { 132 | workspaceRoot := os.Getenv(EnvvarWorkspaceRoot) 133 | if workspaceRoot == "" { 134 | var err error 135 | workspaceRoot, err = leeway.DiscoverWorkspaceRoot() 136 | if err != nil { 137 | log.WithError(err).Debug("cannot determine workspace root - defaulting to .") 138 | workspaceRoot = "." 139 | } else { 140 | log.WithField("workspace", workspaceRoot).Debug("found workspace root") 141 | } 142 | } 143 | 144 | rootCmd.PersistentFlags().StringVarP(&workspace, "workspace", "w", workspaceRoot, "Workspace root") 145 | rootCmd.PersistentFlags().StringArrayVarP(&buildArgs, "build-arg", "D", []string{}, "pass arguments to BUILD files") 146 | rootCmd.PersistentFlags().StringVar(&variant, "variant", "", "selects a package variant") 147 | rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enables verbose logging") 148 | rootCmd.PersistentFlags().Bool("dut", false, "used for testing only - doesn't actually do anything") 149 | } 150 | 151 | func getWorkspace() (leeway.Workspace, error) { 152 | args, err := getBuildArgs() 153 | if err != nil { 154 | return leeway.Workspace{}, err 155 | } 156 | 157 | return leeway.FindWorkspace(workspace, args, variant, os.Getenv("LEEWAY_PROVENANCE_KEYPATH")) 158 | } 159 | 160 | func getBuildArgs() (leeway.Arguments, error) { 161 | if len(buildArgs) == 0 { 162 | return nil, nil 163 | } 164 | 165 | res := make(leeway.Arguments) 166 | for _, arg := range buildArgs { 167 | segs := strings.Split(arg, "=") 168 | if len(segs) < 2 { 169 | return nil, xerrors.Errorf("invalid build argument (format is key=value): %s", arg) 170 | } 171 | res[segs[0]] = strings.Join(segs[1:], "=") 172 | } 173 | return res, nil 174 | } 175 | 176 | func addExperimentalCommand(parent, child *cobra.Command) { 177 | if os.Getenv("LEEWAY_EXPERIMENTAL") != "true" { 178 | return 179 | } 180 | 181 | parent.AddCommand(child) 182 | } 183 | -------------------------------------------------------------------------------- /cmd/run.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | "golang.org/x/sync/errgroup" 9 | ) 10 | 11 | // runCmd represents the version command 12 | var runCmd = &cobra.Command{ 13 | Use: "run [scripts]", 14 | Short: "Executes one or more scripts in parallel", 15 | Long: `Executes one or more scripts in parallel 16 | All scripts will run to completion, regardless of whether or not the other scripts exit with errors. 17 | 18 | Should any of the scripts fail Leeway will exit with an exit code of 1 once all scripts are done executing. 19 | `, 20 | Args: cobra.MinimumNArgs(1), 21 | Run: func(cmd *cobra.Command, args []string) { 22 | g := new(errgroup.Group) 23 | for _, scriptName := range args { 24 | scriptName := scriptName 25 | g.Go(func() error { 26 | _, _, script, _ := getTarget([]string{scriptName}, true) 27 | if script == nil { 28 | return errors.New("run needs a script") 29 | } 30 | opts, _ := getBuildOpts(cmd) 31 | return script.Run(opts...) 32 | }) 33 | } 34 | err := g.Wait() 35 | if err != nil { 36 | log.Fatal(err) 37 | } 38 | }, 39 | } 40 | 41 | func init() { 42 | rootCmd.AddCommand(runCmd) 43 | addBuildFlags(runCmd) 44 | } 45 | -------------------------------------------------------------------------------- /cmd/sbom-export.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/gitpod-io/leeway/pkg/leeway" 11 | "github.com/gitpod-io/leeway/pkg/leeway/cache" 12 | log "github.com/sirupsen/logrus" 13 | "github.com/spf13/cobra" 14 | ) 15 | 16 | // sbomExportCmd represents the sbom export command 17 | var sbomExportCmd = &cobra.Command{ 18 | Use: "export [package]", 19 | Short: "Exports the SBOM of a (previously built) package", 20 | Long: `Exports the SBOM of a (previously built) package. 21 | 22 | When used with --with-dependencies, it exports SBOMs for the package and all its dependencies 23 | to the specified output directory. 24 | 25 | If no package is specified, the workspace's default target is used.`, 26 | Args: cobra.MaximumNArgs(1), 27 | Run: func(cmd *cobra.Command, args []string) { 28 | // Get the package 29 | _, pkg, _, _ := getTarget(args, false) 30 | if pkg == nil { 31 | log.Fatal("sbom export requires a package or a default target in the workspace") 32 | } 33 | 34 | // Get build options and cache 35 | _, localCache := getBuildOpts(cmd) 36 | 37 | // Get output format and file 38 | format, _ := cmd.Flags().GetString("format") 39 | outputFile, _ := cmd.Flags().GetString("output") 40 | withDependencies, _ := cmd.Flags().GetBool("with-dependencies") 41 | outputDir, _ := cmd.Flags().GetString("output-dir") 42 | 43 | // Validate format using the utility function 44 | formatValid, validFormats := leeway.ValidateSBOMFormat(format) 45 | if !formatValid { 46 | log.Fatalf("Unsupported format: %s. Supported formats are: %s", format, strings.Join(validFormats, ", ")) 47 | } 48 | 49 | // Validate flags for dependency export 50 | if withDependencies { 51 | if outputDir == "" { 52 | log.Fatal("--output-dir is required when using --with-dependencies") 53 | } 54 | if outputFile != "" { 55 | log.Fatal("--output and --output-dir cannot be used together") 56 | } 57 | } 58 | 59 | var allpkg []*leeway.Package 60 | allpkg = append(allpkg, pkg) 61 | 62 | if withDependencies { 63 | // Get all dependencies 64 | deps := pkg.GetTransitiveDependencies() 65 | 66 | // Skip ephemeral packages as they're not meant to be cached 67 | for _, p := range deps { 68 | if p.Ephemeral { 69 | log.Infof("Skipping vulnerability scan for ephemeral package %s\n", p.FullName()) 70 | continue 71 | } 72 | allpkg = append(allpkg, p) 73 | } 74 | 75 | log.Infof("Exporting SBOMs for %s and %d dependencies to %s", pkg.FullName(), len(allpkg)-1, outputDir) 76 | } 77 | 78 | for _, p := range allpkg { 79 | var outputPath string 80 | if outputFile == "" { 81 | safeFilename := p.FilesystemSafeName() 82 | outputPath = filepath.Join(outputDir, safeFilename+leeway.GetSBOMFileExtension(format)) 83 | } else { 84 | outputPath = outputFile 85 | } 86 | exportSBOM(p, localCache, outputPath, format) 87 | } 88 | }, 89 | } 90 | 91 | func init() { 92 | sbomExportCmd.Flags().String("format", "cyclonedx", "SBOM format to export (cyclonedx, spdx, syft)") 93 | sbomExportCmd.Flags().StringP("output", "o", "", "Output file (defaults to stdout)") 94 | sbomExportCmd.Flags().Bool("with-dependencies", false, "Export SBOMs for the package and all its dependencies") 95 | sbomExportCmd.Flags().String("output-dir", "", "Output directory for exporting multiple SBOMs (required with --with-dependencies)") 96 | 97 | sbomCmd.AddCommand(sbomExportCmd) 98 | addBuildFlags(sbomExportCmd) 99 | } 100 | 101 | // exportSBOM extracts and writes an SBOM from a package's cached archive. 102 | // It retrieves the package from the cache, creates the output file if needed, 103 | // and extracts the SBOM in the specified format. If outputFile is empty, 104 | // the SBOM is written to stdout. 105 | func exportSBOM(pkg *leeway.Package, localCache cache.LocalCache, outputFile string, format string) { 106 | pkgFN := GetPackagePath(pkg, localCache) 107 | 108 | var output io.Writer = os.Stdout 109 | 110 | // Create directory if it doesn't exist 111 | if dir := filepath.Dir(outputFile); dir != "" { 112 | if err := os.MkdirAll(dir, 0755); err != nil { 113 | log.WithError(err).Fatalf("cannot create output directory %s", dir) 114 | } 115 | } 116 | 117 | file, err := os.Create(outputFile) 118 | if err != nil { 119 | log.WithError(err).Fatalf("cannot create output file %s", outputFile) 120 | } 121 | defer file.Close() 122 | output = file 123 | 124 | // Extract and output the SBOM 125 | err = leeway.AccessSBOMInCachedArchive(pkgFN, format, func(sbomReader io.Reader) error { 126 | log.Infof("Exporting SBOM in %s format", format) 127 | _, err := io.Copy(output, sbomReader) 128 | return err 129 | }) 130 | 131 | if err != nil { 132 | if err == leeway.ErrNoSBOMFile { 133 | log.Fatalf("no SBOM file found in package %s", pkg.FullName()) 134 | } 135 | log.WithError(err).Fatal("cannot extract SBOM") 136 | } 137 | 138 | if outputFile != "" { 139 | log.Infof("SBOM exported to %s", outputFile) 140 | } 141 | } 142 | 143 | // GetPackagePath retrieves the filesystem path to a package's cached archive. 144 | // It first checks the local cache, and if not found, attempts to download 145 | // the package from the remote cache. This function verifies that SBOM is enabled 146 | // in the workspace settings and returns the path to the package archive. 147 | // If the package cannot be found in either cache, it exits with a fatal error. 148 | func GetPackagePath(pkg *leeway.Package, localCache cache.LocalCache) (packagePath string) { 149 | // Check if SBOM is enabled in workspace settings 150 | if !pkg.C.W.SBOM.Enabled { 151 | log.Fatal("SBOM export/scan requires sbom.enabled=true in workspace settings") 152 | } 153 | 154 | if log.IsLevelEnabled(log.DebugLevel) { 155 | v, err := pkg.Version() 156 | if err != nil { 157 | log.WithError(err).Fatal("error getting version") 158 | } 159 | log.Debugf("Exporting SBOM of package %s (version %s)", pkg.FullName(), v) 160 | } 161 | 162 | // Get package location in local cache 163 | pkgFN, ok := localCache.Location(pkg) 164 | if !ok { 165 | // Package not found in local cache, check if it's in the remote cache 166 | log.Debugf("Package %s not found in local cache, checking remote cache", pkg.FullName()) 167 | 168 | remoteCache := getRemoteCache() 169 | remoteCache = &pullOnlyRemoteCache{C: remoteCache} 170 | 171 | // Convert to cache.Package interface 172 | pkgsToCheck := []cache.Package{pkg} 173 | 174 | if log.IsLevelEnabled(log.DebugLevel) { 175 | v, err := pkgsToCheck[0].Version() 176 | if err != nil { 177 | log.WithError(err).Fatal("error getting version") 178 | } 179 | log.Debugf("Checking remote of package %s (version %s)", pkgsToCheck[0].FullName(), v) 180 | } 181 | 182 | // Check if the package exists in the remote cache 183 | existingPkgs, err := remoteCache.ExistingPackages(context.Background(), pkgsToCheck) 184 | if err != nil { 185 | log.WithError(err).Warnf("Failed to check if package %s exists in remote cache", pkg.FullName()) 186 | log.Fatalf("%s is not built", pkg.FullName()) 187 | } else { 188 | _, existsInRemote := existingPkgs[pkg] 189 | if existsInRemote { 190 | log.Infof("Package %s found in remote cache, downloading...", pkg.FullName()) 191 | 192 | // Download the package from the remote cache 193 | err := remoteCache.Download(context.Background(), localCache, pkgsToCheck) 194 | if err != nil { 195 | log.WithError(err).Fatalf("Failed to download package %s from remote cache", pkg.FullName()) 196 | } 197 | 198 | // Check if the download was successful 199 | pkgFN, ok = localCache.Location(pkg) 200 | if !ok { 201 | log.Fatalf("Failed to download package %s from remote cache", pkg.FullName()) 202 | } 203 | 204 | log.Infof("Successfully downloaded package %s from remote cache", pkg.FullName()) 205 | } else { 206 | log.Fatalf("%s is not built", pkg.FullName()) 207 | } 208 | } 209 | } 210 | return pkgFN 211 | } 212 | -------------------------------------------------------------------------------- /cmd/sbom-scan.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/gitpod-io/leeway/pkg/leeway" 7 | log "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | // sbomScanCmd represents the sbom scan command 12 | var sbomScanCmd = &cobra.Command{ 13 | Use: "scan [package]", 14 | Short: "Scans a package's SBOM for vulnerabilities", 15 | Long: `Scans a package's SBOM for vulnerabilities and exports the results to a specified directory. 16 | 17 | This command uses existing SBOM files from previously built packages. It checks if SBOM is enabled 18 | in the workspace settings. If not, it aborts. The scan results are exported to the directory 19 | specified by the --output-dir flag. 20 | 21 | When used with --with-dependencies, it scans the package and all its dependencies for vulnerabilities. 22 | 23 | If no package is specified, the workspace's default target is used.`, 24 | Args: cobra.MaximumNArgs(1), 25 | Run: func(cmd *cobra.Command, args []string) { 26 | // Get the package 27 | _, pkg, _, _ := getTarget(args, false) 28 | if pkg == nil { 29 | log.Fatal("sbom scan requires a package or a default target in the workspace") 30 | } 31 | 32 | // Get cache 33 | _, localCache := getBuildOpts(cmd) 34 | 35 | // Get output directory 36 | outputDir, _ := cmd.Flags().GetString("output-dir") 37 | if outputDir == "" { 38 | log.Fatal("--output-dir is required") 39 | } 40 | 41 | // Create output directory if it doesn't exist 42 | if err := os.MkdirAll(outputDir, 0755); err != nil { 43 | log.WithError(err).Fatalf("cannot create output directory %s", outputDir) 44 | } 45 | 46 | // Get with-dependencies flag 47 | withDependencies, _ := cmd.Flags().GetBool("with-dependencies") 48 | 49 | var allpkg []*leeway.Package 50 | allpkg = append(allpkg, pkg) 51 | 52 | if withDependencies { 53 | // Get all dependencies 54 | deps := pkg.GetTransitiveDependencies() 55 | log.Infof("Scanning SBOMs for %s and %d dependencies to %s", pkg.FullName(), len(deps), outputDir) 56 | 57 | // Skip ephemeral packages as they're not meant to be cached 58 | var filteredDeps []*leeway.Package 59 | for _, p := range deps { 60 | if p.Ephemeral { 61 | log.Infof("Skipping vulnerability scan for ephemeral package %s\n", p.FullName()) 62 | continue 63 | } 64 | filteredDeps = append(filteredDeps, p) 65 | } 66 | 67 | allpkg = append(allpkg, filteredDeps...) 68 | } 69 | 70 | // Download packages from remote cache when needed 71 | for _, p := range allpkg { 72 | GetPackagePath(p, localCache) 73 | } 74 | 75 | if err := leeway.ScanAllPackagesForVulnerabilities(localCache, allpkg, outputDir); err != nil { 76 | log.WithError(err).Fatalf("Failed to scan package %s for vulnerabilities", pkg.FullName()) 77 | } 78 | 79 | if withDependencies { 80 | log.Infof("Vulnerability scan completed for package %s and its dependencies", pkg.FullName()) 81 | } else { 82 | log.Infof("Vulnerability scan completed for package %s", pkg.FullName()) 83 | } 84 | log.Infof("Scan results exported to %s", outputDir) 85 | 86 | // If we have failOn configured, the ScanPackageForVulnerabilities function will have already 87 | // returned an error if vulnerabilities at those severity levels were found 88 | }, 89 | } 90 | 91 | func init() { 92 | sbomScanCmd.Flags().String("output-dir", "", "Directory to export scan results (required)") 93 | if err := sbomScanCmd.MarkFlagRequired("output-dir"); err != nil { 94 | log.WithError(err).Fatal("failed to mark output-dir flag as required") 95 | } 96 | sbomScanCmd.Flags().Bool("with-dependencies", false, "Scan the package and all its dependencies") 97 | 98 | sbomCmd.AddCommand(sbomScanCmd) 99 | addBuildFlags(sbomScanCmd) 100 | } 101 | -------------------------------------------------------------------------------- /cmd/sbom.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | // sbomCmd represents the sbom command 8 | var sbomCmd = &cobra.Command{ 9 | Use: "sbom ", 10 | Short: "Helpful commands for working with Software Bill of Materials (SBOM)", 11 | Long: `Helpful commands for working with Software Bill of Materials (SBOM). 12 | 13 | The subcommands (export, scan) can be used with or without specifying a package. 14 | If no package is specified, the workspace's default target is used.`, 15 | Args: cobra.MinimumNArgs(1), 16 | } 17 | 18 | func init() { 19 | rootCmd.AddCommand(sbomCmd) 20 | } 21 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/gitpod-io/leeway/pkg/leeway" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | // versionCmd represents the version command 11 | var versionCmd = &cobra.Command{ 12 | Use: "version", 13 | Short: "Prints the version of this leeway build", 14 | Run: func(cmd *cobra.Command, args []string) { 15 | fmt.Println(leeway.Version) 16 | }, 17 | } 18 | 19 | func init() { 20 | rootCmd.AddCommand(versionCmd) 21 | } 22 | -------------------------------------------------------------------------------- /cmd/vet.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | 9 | "github.com/gitpod-io/leeway/pkg/prettyprint" 10 | "github.com/gitpod-io/leeway/pkg/vet" 11 | ) 12 | 13 | // versionCmd represents the version command 14 | var vetCmd = &cobra.Command{ 15 | Use: "vet [ls]", 16 | Short: "Validates the leeway workspace", 17 | RunE: func(cmd *cobra.Command, args []string) error { 18 | w := getWriterFromFlags(cmd) 19 | if len(args) > 0 && args[0] == "ls" { 20 | if w.FormatString == "" && w.Format == prettyprint.TemplateFormat { 21 | w.FormatString = `{{ range . -}} 22 | {{ .Name }}{{"\t"}}{{ .Description }} 23 | {{ end }}` 24 | } 25 | err := w.Write(vet.Checks()) 26 | if err != nil { 27 | return err 28 | } 29 | } 30 | 31 | ws, err := getWorkspace() 32 | if err != nil { 33 | return err 34 | } 35 | 36 | var opts []vet.RunOpt 37 | if checks, _ := cmd.Flags().GetStringArray("checks"); len(checks) > 0 { 38 | opts = append(opts, vet.WithChecks(checks)) 39 | } 40 | if pkgs, _ := cmd.Flags().GetStringArray("packages"); len(pkgs) > 0 { 41 | idx := make(vet.StringSet) 42 | for _, p := range pkgs { 43 | idx[p] = struct{}{} 44 | } 45 | opts = append(opts, vet.OnPackages(idx)) 46 | } 47 | if comps, _ := cmd.Flags().GetStringArray("components"); len(comps) > 0 { 48 | idx := make(vet.StringSet) 49 | for _, p := range comps { 50 | idx[p] = struct{}{} 51 | } 52 | opts = append(opts, vet.OnComponents(idx)) 53 | } 54 | 55 | findings, errs := vet.Run(ws, opts...) 56 | if ignoreWarnings, _ := cmd.Flags().GetBool("ignore-warnings"); ignoreWarnings { 57 | n := 0 58 | for _, x := range findings { 59 | if x.Error { 60 | findings[n] = x 61 | n++ 62 | } 63 | } 64 | findings = findings[:n] 65 | } 66 | 67 | if len(errs) != 0 { 68 | for _, err := range errs { 69 | log.Error(err.Error()) 70 | } 71 | return nil 72 | } 73 | 74 | if w.FormatString == "" && w.Format == prettyprint.TemplateFormat { 75 | w.FormatString = `{{ range . }} 76 | {{"\033"}}[90m{{ if .Package -}}📦{{"\t"}}{{ .Package.FullName }}{{ else if .Component }}🗃️{{"\t"}}{{ .Component.Name }}{{ end }} 77 | ✔️ {{ .Check }}{{"\033"}}[0m 78 | {{ if .Error -}}❌{{ else }}⚠️{{ end -}}{{"\t"}}{{ .Description }} 79 | {{ end }}` 80 | } 81 | err = w.Write(findings) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | if len(findings) == 0 { 87 | os.Exit(0) 88 | } else { 89 | os.Exit(128) 90 | } 91 | 92 | return nil 93 | }, 94 | } 95 | 96 | func init() { 97 | rootCmd.AddCommand(vetCmd) 98 | 99 | vetCmd.Flags().StringArray("checks", nil, "run these checks only") 100 | vetCmd.Flags().StringArray("packages", nil, "run checks on these packages only") 101 | vetCmd.Flags().StringArray("components", nil, "run checks on these components only") 102 | vetCmd.Flags().Bool("ignore-warnings", false, "ignores all warnings") 103 | addFormatFlags(vetCmd) 104 | } 105 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | precision: 2 3 | round: down 4 | 5 | status: 6 | project: 7 | default: 8 | informational: true 9 | target: auto 10 | patch: 11 | default: 12 | informational: true 13 | target: auto 14 | 15 | comment: 16 | layout: "header, diff" 17 | behavior: default # update if exists else create new 18 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gitpod-io/leeway/bfe3f2198f83f71afc51894d1bad59669bd95c64/logo.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/gitpod-io/leeway/cmd" 4 | 5 | func main() { 6 | cmd.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /pkg/doublestar/doublestar.go: -------------------------------------------------------------------------------- 1 | package doublestar 2 | 3 | import ( 4 | "path/filepath" 5 | "strings" 6 | 7 | "github.com/karrick/godirwalk" 8 | ) 9 | 10 | // IgnoreFunc checks if a path ought to be ignored 11 | type IgnoreFunc func(path string) bool 12 | 13 | // IgnoreNone ignores nothing 14 | var IgnoreNone IgnoreFunc = func(path string) bool { return false } 15 | 16 | // IgnoreStrings ignores all paths which contain one of the ignores substrings 17 | func IgnoreStrings(ignores []string) IgnoreFunc { 18 | return func(path string) bool { 19 | for _, ptn := range ignores { 20 | if ptn == "" { 21 | continue 22 | } 23 | if strings.Contains(path, ptn) { 24 | return true 25 | } 26 | } 27 | return false 28 | } 29 | } 30 | 31 | // Glob finds all files that match the pattern and not the ignore func 32 | func Glob(base, pattern string, ignore IgnoreFunc) ([]string, error) { 33 | var res []string 34 | err := godirwalk.Walk(base, &godirwalk.Options{ 35 | Callback: func(osPathname string, directoryEntry *godirwalk.Dirent) error { 36 | if ignore != nil && ignore(osPathname) { 37 | if directoryEntry.IsDir() { 38 | return filepath.SkipDir 39 | } 40 | return nil 41 | } 42 | 43 | path := strings.TrimPrefix(osPathname, base+"/") 44 | m, err := Match(pattern, path) 45 | if err != nil { 46 | return err 47 | } 48 | if m { 49 | res = append(res, osPathname) 50 | } 51 | return nil 52 | }, 53 | FollowSymbolicLinks: true, 54 | Unsorted: true, 55 | ErrorCallback: func(path string, err error) godirwalk.ErrorAction { 56 | return godirwalk.SkipNode 57 | }, 58 | }) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | return res, nil 64 | } 65 | 66 | // Match matches the same patterns as filepath.Match except it can also match 67 | // an arbitrary number of path segments using ** 68 | func Match(pattern, path string) (matches bool, err error) { 69 | if path == pattern { 70 | return true, nil 71 | } 72 | 73 | var ( 74 | patterns = strings.Split(filepath.ToSlash(pattern), "/") 75 | paths = strings.Split(filepath.ToSlash(path), "/") 76 | ) 77 | return match(patterns, paths) 78 | } 79 | 80 | func match(patterns, paths []string) (matches bool, err error) { 81 | var pathIndex int 82 | for patternIndex := 0; patternIndex < len(patterns); patternIndex++ { 83 | pattern := patterns[patternIndex] 84 | if patternIndex >= len(paths) { 85 | // pattern is longer than path - path can't match 86 | // TODO: what if the last pattern segment is ** 87 | return false, nil 88 | } 89 | 90 | path := paths[pathIndex] 91 | if pattern == path { 92 | // path and pattern segment match exactly - consume the path segment 93 | pathIndex++ 94 | continue 95 | } 96 | 97 | if pattern == "**" { 98 | if patternIndex == len(patterns)-1 { 99 | // this is the last pattern segment, hence we consume the remainder of the path. 100 | return true, nil 101 | } 102 | 103 | // this segment consumes all path segments until the next pattern segment 104 | nextPattern := patterns[patternIndex+1] 105 | if nextPattern == "**" { 106 | // next pattern is a doublestar, too. Hence we just consume this path segment 107 | // and let the next doublestar do the work. 108 | continue 109 | } 110 | 111 | // we consume one path segment after the other and check if the remainder of the pattern 112 | // matches the remainder of the path 113 | for pi := pathIndex; pi < len(paths); pi++ { 114 | m, err := match(patterns[patternIndex+1:], paths[pi:]) 115 | if err != nil { 116 | return false, err 117 | } 118 | if m { 119 | return true, nil 120 | } 121 | } 122 | // none of the remainder matched 123 | return false, nil 124 | } 125 | 126 | match, err := filepath.Match(pattern, path) 127 | if err != nil { 128 | return false, err 129 | } 130 | if match { 131 | pathIndex++ 132 | continue 133 | } 134 | 135 | // did not find a match - we're done here 136 | return false, nil 137 | } 138 | 139 | // we made it through the whole pattern, which means it matches alright 140 | return true, nil 141 | } 142 | -------------------------------------------------------------------------------- /pkg/doublestar/doublestar_test.go: -------------------------------------------------------------------------------- 1 | package doublestar_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/gitpod-io/leeway/pkg/doublestar" 8 | ) 9 | 10 | func TestMatch(t *testing.T) { 11 | tests := []struct { 12 | Pattern string 13 | Path string 14 | Match bool 15 | }{ 16 | {"**", "/", true}, 17 | {"**/*.go", "foo.go", true}, 18 | {"**/foo.go", "foo.go", true}, 19 | {"**/BUILD.yaml", "fixtures/scripts/BUILD.yaml", true}, 20 | {"**/*.go", "a/b/c/foo.go", true}, 21 | {"**/*.go", "/c/foo.go", true}, 22 | {"**/*.go", "a/b/c/foo.txt", false}, 23 | {"**/*.go", "a/b/c", false}, 24 | {"**/*.go", "/a/b/c", false}, 25 | {"/a/b/**", "/a/b/c", true}, 26 | {"/a/b/**", "/a/b/c/d/e/f/g", true}, 27 | {"/a/b/**", "/a/b", false}, 28 | {"/a/b/**", "a/b/c", false}, 29 | {"/a/b/**/c", "/a/b/c", true}, 30 | {"/a/b/**/c", "/a/b/1/2/3/4/c", true}, 31 | {"/a/b/**/c/*.go", "/a/b/1/2/3/4/c/foo.go", true}, 32 | {"/a/b/**/c/*.go", "/a/b/1/2/3/4/c/foo.txt", false}, 33 | {"/a/b/**/**/c", "/a/b/1/2/3/4/c", true}, 34 | {"/a/b/**/**/c", "/a/b/1/c", true}, 35 | {"/a/b/**/c/**/d", "/a/b/1/c/2/d", true}, 36 | {"/a/b/**/c/**/d", "/a/b/1/c/2", false}, 37 | {"*/*.go", "src/foo.go", true}, 38 | } 39 | for i, test := range tests { 40 | t.Run(fmt.Sprintf("%03d_%s_%s", i, test.Pattern, test.Path), func(t *testing.T) { 41 | match, err := doublestar.Match(test.Pattern, test.Path) 42 | if err != nil { 43 | t.Fatalf("unexpected error: %q", err) 44 | } 45 | if match != test.Match { 46 | t.Errorf("unexpected match: expected %v, got %v", test.Match, match) 47 | } 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /pkg/leeway/build_darwin.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "fmt" 5 | "os/exec" 6 | "strings" 7 | 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | func checkForCpCompatibility() error { 12 | out, err := exec.Command("cp", "--help").CombinedOutput() 13 | if err != nil && !strings.Contains(err.Error(), "exit") { 14 | log.WithError(err).Debug("cannot check if cp is compatible") 15 | // if cp is not compatible we'll fail later in the build, 16 | // but maybe it is and we don't want to fail here for no good reason. 17 | return nil 18 | } 19 | 20 | if strings.Contains(string(out), "--parents") { 21 | // we're good 22 | return nil 23 | } 24 | 25 | return fmt.Errorf("leeway requires a GNU-compatible cp. Please install using `brew install coreutils`; make sure you update your PATH after installing.") 26 | } 27 | 28 | func executeCommandsForPackageSafe(buildctx *buildContext, p *Package, wd string, commands [][]string) error { 29 | return fmt.Errorf("not implemented") 30 | } 31 | -------------------------------------------------------------------------------- /pkg/leeway/build_internal_test.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | ) 8 | 9 | func TestParseGoCoverOutput(t *testing.T) { 10 | type Expectation struct { 11 | Error string 12 | Coverage int 13 | FuncsWithoutTest int 14 | FuncsWithTest int 15 | } 16 | tests := []struct { 17 | Name string 18 | Input string 19 | Expectation Expectation 20 | }{ 21 | { 22 | Name: "empty", 23 | }, 24 | { 25 | Name: "valid", 26 | Input: `github.com/gitpod-io/leeway/store.go:165: Get 100.0% 27 | github.com/gitpod-io/leeway/store.go:173: Set 100.0% 28 | github.com/gitpod-io/leeway/store.go:178: Delete 100.0% 29 | github.com/gitpod-io/leeway/store.go:183: Scan 80.0% 30 | github.com/gitpod-io/leeway/store.go:194: Close 0.0% 31 | github.com/gitpod-io/leeway/store.go:206: Upsert 0.0%`, 32 | Expectation: Expectation{ 33 | Coverage: 63, 34 | FuncsWithoutTest: 2, 35 | FuncsWithTest: 4, 36 | }, 37 | }, 38 | } 39 | 40 | for _, test := range tests { 41 | t.Run(test.Name, func(t *testing.T) { 42 | var act Expectation 43 | 44 | var err error 45 | act.Coverage, act.FuncsWithoutTest, act.FuncsWithTest, err = parseGoCoverOutput(test.Input) 46 | if err != nil { 47 | act.Error = err.Error() 48 | } 49 | 50 | if diff := cmp.Diff(test.Expectation, act); diff != "" { 51 | t.Errorf("parseGoCoverOutput() mismatch (-want +got):\n%s", diff) 52 | } 53 | }) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pkg/leeway/build_linux.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/opencontainers/runc/libcontainer/specconv" 13 | "github.com/opencontainers/runtime-spec/specs-go" 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | func checkForCpCompatibility() error { 18 | // we're on linux - just assume it's GNU cp 19 | return nil 20 | } 21 | 22 | func executeCommandsForPackageSafe(buildctx *buildContext, p *Package, wd string, commands [][]string) error { 23 | tmpdir, err := os.MkdirTemp("", "leeway-*") 24 | if err != nil { 25 | return err 26 | } 27 | 28 | jc, err := json.Marshal(commands) 29 | if err != nil { 30 | return err 31 | } 32 | commandsFN := filepath.Join(tmpdir, "commands") 33 | err = os.WriteFile(commandsFN, []byte(base64.StdEncoding.EncodeToString(jc)), 0644) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | if !log.IsLevelEnabled(log.DebugLevel) { 39 | defer os.RemoveAll(tmpdir) 40 | } 41 | 42 | log.WithField("tmpdir", tmpdir).WithField("package", p.FullName()).Debug("preparing build runc environment") 43 | err = os.MkdirAll(filepath.Join(tmpdir, "rootfs"), 0755) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | version, err := p.Version() 49 | if err != nil { 50 | return err 51 | } 52 | name := fmt.Sprintf("b%s", version) 53 | 54 | spec := specconv.Example() 55 | specconv.ToRootless(spec) 56 | 57 | // we assemble the root filesystem from the outside world 58 | for _, d := range []string{"home", "bin", "dev", "etc", "lib", "lib64", "opt", "sbin", "sys", "usr", "var"} { 59 | spec.Mounts = append(spec.Mounts, specs.Mount{ 60 | Destination: "/" + d, 61 | Source: "/" + d, 62 | Type: "bind", 63 | Options: []string{"rbind", "rprivate"}, 64 | }) 65 | } 66 | 67 | spec.Mounts = append(spec.Mounts, specs.Mount{Destination: "/build", Source: wd, Type: "bind", Options: []string{"bind", "private"}}) 68 | spec.Mounts = append(spec.Mounts, specs.Mount{Destination: "/commands", Source: commandsFN, Type: "bind", Options: []string{"bind", "private"}}) 69 | 70 | for _, p := range []string{"tmp", "root"} { 71 | fn := filepath.Join(tmpdir, p) 72 | err = os.MkdirAll(fn, 0777) 73 | if err != nil { 74 | return err 75 | } 76 | spec.Mounts = append(spec.Mounts, specs.Mount{Destination: "/" + p, Source: fn, Type: "bind", Options: []string{"bind", "private"}}) 77 | } 78 | 79 | buildCache, _ := buildctx.LocalCache.Location(p) 80 | buildCache = filepath.Dir(buildCache) 81 | spec.Mounts = append(spec.Mounts, specs.Mount{Destination: buildCache, Source: buildCache, Type: "bind", Options: []string{"bind", "private"}}) 82 | 83 | self, err := os.Executable() 84 | if err != nil { 85 | return err 86 | } 87 | spec.Mounts = append(spec.Mounts, specs.Mount{Destination: "/leeway", Source: self, Type: "bind", Options: []string{"bind", "private"}}) 88 | 89 | if p := os.Getenv("GOPATH"); p != "" { 90 | spec.Mounts = append(spec.Mounts, specs.Mount{Destination: p, Source: p, Type: "bind", Options: []string{"bind", "private"}}) 91 | } 92 | if p := os.Getenv("GOROOT"); p != "" { 93 | spec.Mounts = append(spec.Mounts, specs.Mount{Destination: p, Source: p, Type: "bind", Options: []string{"bind", "private"}}) 94 | } 95 | if p := os.Getenv("DOCKER_HOST"); strings.HasPrefix(p, "file://") { 96 | p = strings.TrimPrefix(p, "file://") 97 | spec.Mounts = append(spec.Mounts, specs.Mount{Destination: p, Source: p, Type: "bind", Options: []string{"bind", "private"}}) 98 | } else if _, err := os.Stat("/var/run/docker.sock"); err == nil { 99 | p = "/var/run/docker.sock" 100 | spec.Mounts = append(spec.Mounts, specs.Mount{Destination: p, Source: p, Type: "bind", Options: []string{"bind", "private"}}) 101 | } 102 | 103 | var env []string 104 | for _, e := range []string{"PATH", "TERM", "GOROOT", "GOPATH"} { 105 | val := os.Getenv(e) 106 | if val == "" { 107 | continue 108 | } 109 | env = append(env, fmt.Sprintf("%s=%s", e, val)) 110 | } 111 | 112 | spec.Hostname = name 113 | spec.Process.Terminal = false 114 | spec.Process.NoNewPrivileges = true 115 | spec.Process.Args = []string{"/leeway", "plumbing", "exec", "/commands"} 116 | if log.IsLevelEnabled(log.DebugLevel) { 117 | spec.Process.Args = append(spec.Process.Args, "--verbose") 118 | 119 | } 120 | spec.Process.Cwd = "/build" 121 | spec.Process.Env = env 122 | 123 | fc, err := json.MarshalIndent(spec, "", " ") 124 | if err != nil { 125 | return err 126 | } 127 | err = os.WriteFile(filepath.Join(tmpdir, "config.json"), fc, 0644) 128 | if err != nil { 129 | return err 130 | } 131 | 132 | args := []string{ 133 | "--root", "state", 134 | "--log-format", "json", 135 | } 136 | if log.IsLevelEnabled(log.DebugLevel) { 137 | args = append(args, "--debug") 138 | } 139 | args = append(args, 140 | "run", name, 141 | ) 142 | 143 | cmd := exec.Command("runc", args...) 144 | cmd.Dir = tmpdir 145 | cmd.Stdout = &reporterStream{R: buildctx.Reporter, P: p, IsErr: false} 146 | cmd.Stderr = &reporterStream{R: buildctx.Reporter, P: p, IsErr: true} 147 | return cmd.Run() 148 | } 149 | -------------------------------------------------------------------------------- /pkg/leeway/build_test.go: -------------------------------------------------------------------------------- 1 | package leeway_test 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/gitpod-io/leeway/pkg/leeway" 9 | "github.com/gitpod-io/leeway/pkg/testutil" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | const dummyDocker = `#!/bin/bash 14 | 15 | POSITIONAL_ARGS=() 16 | 17 | while [[ $# -gt 0 ]]; do 18 | case $1 in 19 | -o) 20 | OUTPUT="$2" 21 | shift # past argument 22 | shift # past value 23 | ;; 24 | inspect) 25 | # Mock docker inspect to return a valid ID 26 | echo '[{"Id":"sha256:1234567890abcdef"}]' 27 | exit 0 28 | ;; 29 | *) 30 | POSITIONAL_ARGS+=("$1") # save positional arg 31 | shift # past argument 32 | ;; 33 | esac 34 | done 35 | 36 | set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters 37 | 38 | if [ "${POSITIONAL_ARGS}" == "save" ]; then 39 | tar cvvfz "${OUTPUT}" -T /dev/null 40 | fi 41 | ` 42 | 43 | // Create a mock for extractImageWithOCILibs to avoid dependency on actual Docker daemon 44 | func init() { 45 | // Override with a simple mock implementation for tests 46 | leeway.ExtractImageWithOCILibs = func(destDir, imgTag string) error { 47 | log.WithFields(log.Fields{ 48 | "image": imgTag, 49 | "destDir": destDir, 50 | }).Info("Mock: Extracting container filesystem") 51 | 52 | // Create required directories 53 | contentDir := filepath.Join(destDir, "content") 54 | if err := os.MkdirAll(contentDir, 0755); err != nil { 55 | return err 56 | } 57 | 58 | // Create a mock file structure similar to what a real extraction would produce 59 | mockFiles := map[string]string{ 60 | filepath.Join(destDir, "imgnames.txt"): imgTag + "\n", 61 | filepath.Join(destDir, "metadata.yaml"): "test: metadata\n", 62 | filepath.Join(destDir, "image-metadata.json"): `{"image":"` + imgTag + `"}`, 63 | filepath.Join(contentDir, "bin/testfile"): "test content", 64 | filepath.Join(contentDir, "README.md"): "# Test Container", 65 | } 66 | 67 | // Create directories for the mock files 68 | for filename := range mockFiles { 69 | if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { 70 | return err 71 | } 72 | } 73 | 74 | // Create the mock files 75 | for filename, content := range mockFiles { 76 | if err := os.WriteFile(filename, []byte(content), 0644); err != nil { 77 | return err 78 | } 79 | } 80 | 81 | return nil 82 | } 83 | } 84 | 85 | func TestBuildDockerDeps(t *testing.T) { 86 | if *testutil.Dut { 87 | pth, err := os.MkdirTemp("", "") 88 | if err != nil { 89 | t.Fatal(err) 90 | } 91 | err = os.WriteFile(filepath.Join(pth, "docker"), []byte(dummyDocker), 0755) 92 | if err != nil { 93 | t.Fatal(err) 94 | } 95 | t.Cleanup(func() { os.RemoveAll(pth) }) 96 | 97 | os.Setenv("PATH", pth+":"+os.Getenv("PATH")) 98 | log.WithField("path", os.Getenv("PATH")).Debug("modified path to use dummy docker") 99 | } 100 | testutil.RunDUT() 101 | 102 | tests := []*testutil.CommandFixtureTest{ 103 | { 104 | Name: "docker dependency", 105 | T: t, 106 | Args: []string{"build", "-v", "-c", "none", "comp:pkg1"}, 107 | StderrSub: "DEP_COMP__PKG0=foobar:1234", 108 | NoStdoutSub: "already built", 109 | ExitCode: 0, 110 | Fixture: &testutil.Setup{ 111 | Components: []testutil.Component{ 112 | { 113 | Location: "comp", 114 | Files: map[string]string{ 115 | "pkg0.Dockerfile": "FROM alpine:latest", 116 | "pkg1.Dockerfile": "FROM ${DEP_COMP__PKG0}", 117 | }, 118 | Packages: []leeway.Package{ 119 | { 120 | PackageInternal: leeway.PackageInternal{ 121 | Name: "pkg0", 122 | Type: leeway.DockerPackage, 123 | }, 124 | Config: leeway.DockerPkgConfig{ 125 | Dockerfile: "pkg0.Dockerfile", 126 | Image: []string{"foobar:1234"}, 127 | Metadata: make(map[string]string), 128 | }, 129 | }, 130 | { 131 | PackageInternal: leeway.PackageInternal{ 132 | Name: "pkg1", 133 | Type: leeway.DockerPackage, 134 | Dependencies: []string{":pkg0"}, 135 | }, 136 | Config: leeway.DockerPkgConfig{ 137 | Dockerfile: "pkg1.Dockerfile", 138 | Metadata: make(map[string]string), 139 | }, 140 | }, 141 | }, 142 | }, 143 | }, 144 | }, 145 | }, 146 | } 147 | 148 | for _, test := range tests { 149 | test.Run() 150 | } 151 | } 152 | 153 | func TestDockerPostProcessing(t *testing.T) { 154 | if *testutil.Dut { 155 | pth, err := os.MkdirTemp("", "") 156 | if err != nil { 157 | t.Fatal(err) 158 | } 159 | err = os.WriteFile(filepath.Join(pth, "docker"), []byte(dummyDocker), 0755) 160 | if err != nil { 161 | t.Fatal(err) 162 | } 163 | t.Cleanup(func() { os.RemoveAll(pth) }) 164 | 165 | os.Setenv("PATH", pth+":"+os.Getenv("PATH")) 166 | log.WithField("path", os.Getenv("PATH")).Debug("modified path to use dummy docker") 167 | } 168 | testutil.RunDUT() 169 | 170 | tests := []*testutil.CommandFixtureTest{ 171 | { 172 | Name: "docker extraction", 173 | T: t, 174 | Args: []string{"build", "-v", "-c", "none", "comp:pkg"}, 175 | StderrSub: "Extracting container filesystem", 176 | ExitCode: 0, 177 | Fixture: &testutil.Setup{ 178 | Components: []testutil.Component{ 179 | { 180 | Location: "comp", 181 | Files: map[string]string{ 182 | "Dockerfile": "FROM alpine:latest", 183 | }, 184 | Packages: []leeway.Package{ 185 | { 186 | PackageInternal: leeway.PackageInternal{ 187 | Name: "pkg", 188 | Type: leeway.DockerPackage, 189 | }, 190 | Config: leeway.DockerPkgConfig{ 191 | Dockerfile: "Dockerfile", 192 | // No Image entry - should trigger extraction 193 | }, 194 | }, 195 | }, 196 | }, 197 | }, 198 | }, 199 | }, 200 | { 201 | Name: "docker content directory structure", 202 | T: t, 203 | Args: []string{"build", "-v", "-c", "none", "comp:content-test"}, 204 | StderrSub: "Container files extracted successfully", 205 | ExitCode: 0, 206 | Fixture: &testutil.Setup{ 207 | Components: []testutil.Component{ 208 | { 209 | Location: "comp", 210 | Files: map[string]string{ 211 | "content.Dockerfile": "FROM alpine:latest\nRUN mkdir -p /test/dir\nRUN echo 'test' > /test/file.txt", 212 | }, 213 | Packages: []leeway.Package{ 214 | { 215 | PackageInternal: leeway.PackageInternal{ 216 | Name: "content-test", 217 | Type: leeway.DockerPackage, 218 | }, 219 | Config: leeway.DockerPkgConfig{ 220 | Dockerfile: "content.Dockerfile", 221 | // No Image entry - should trigger extraction 222 | }, 223 | }, 224 | }, 225 | }, 226 | }, 227 | }, 228 | }, 229 | } 230 | 231 | for _, test := range tests { 232 | test.Run() 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /pkg/leeway/cache/local/fs.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/gitpod-io/leeway/pkg/leeway/cache" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | // FilesystemCache implements a flat folder cache 13 | type FilesystemCache struct { 14 | Origin string 15 | } 16 | 17 | // NewFilesystemCache creates a new filesystem cache 18 | func NewFilesystemCache(location string) (*FilesystemCache, error) { 19 | err := os.MkdirAll(location, 0755) 20 | if err != nil { 21 | return nil, fmt.Errorf("failed to create cache directory: %w", err) 22 | } 23 | 24 | return &FilesystemCache{location}, nil 25 | } 26 | 27 | // Location computes the name of a packages build result artifact. 28 | // Returns ok == true if that build artifact actually exists. 29 | func (fsc *FilesystemCache) Location(pkg cache.Package) (path string, exists bool) { 30 | version, err := pkg.Version() 31 | if err != nil { 32 | log.WithError(err).WithField("package", pkg.FullName()).Warn("Failed to get package version") 33 | return "", false 34 | } 35 | 36 | // Ensure the cache directory exists first - do this unconditionally 37 | if err := os.MkdirAll(fsc.Origin, 0755); err != nil { 38 | log.WithError(err).WithField("dir", fsc.Origin).Warn("Failed to create cache directory") 39 | return "", false 40 | } 41 | 42 | // Check for .tar.gz file first 43 | gzPath := filepath.Join(fsc.Origin, fmt.Sprintf("%s.tar.gz", version)) 44 | if fileExists(gzPath) { 45 | return gzPath, true 46 | } 47 | 48 | // Fall back to .tar file 49 | tarPath := filepath.Join(fsc.Origin, fmt.Sprintf("%s.tar", version)) 50 | exists = fileExists(tarPath) 51 | 52 | // Always ensure the parent directory exists for this path 53 | // This is critical for download operations to succeed 54 | if err := os.MkdirAll(filepath.Dir(tarPath), 0755); err != nil { 55 | log.WithError(err).WithField("dir", filepath.Dir(tarPath)).Warn("Failed to create directory for package") 56 | } 57 | 58 | return tarPath, exists 59 | } 60 | 61 | // fileExists checks if a file exists and is not a directory 62 | func fileExists(filename string) bool { 63 | info, err := os.Stat(filename) 64 | if err != nil { 65 | return false 66 | } 67 | return !info.IsDir() 68 | } 69 | -------------------------------------------------------------------------------- /pkg/leeway/cache/local/fs_test.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/gitpod-io/leeway/pkg/leeway/cache" 9 | "github.com/google/go-cmp/cmp" 10 | ) 11 | 12 | type mockPackage struct { 13 | version string 14 | err error 15 | } 16 | 17 | func (m mockPackage) Version() (string, error) { 18 | return m.version, m.err 19 | } 20 | 21 | func (m mockPackage) FullName() string { 22 | return "mock-package" 23 | } 24 | 25 | func TestNewFilesystemCache(t *testing.T) { 26 | t.Parallel() 27 | 28 | type Expectation struct { 29 | Error string 30 | } 31 | 32 | tests := []struct { 33 | Name string 34 | Location string 35 | Expectation Expectation 36 | }{ 37 | { 38 | Name: "valid location", 39 | Location: "testdata/cache", 40 | Expectation: Expectation{ 41 | Error: "", 42 | }, 43 | }, 44 | { 45 | Name: "invalid location", 46 | Location: "/proc/invalid/location", 47 | Expectation: Expectation{ 48 | Error: "failed to create cache directory:", 49 | }, 50 | }, 51 | } 52 | 53 | for _, test := range tests { 54 | test := test 55 | t.Run(test.Name, func(t *testing.T) { 56 | t.Parallel() 57 | var act Expectation 58 | 59 | _, err := NewFilesystemCache(test.Location) 60 | if err != nil { 61 | act.Error = err.Error()[:len(test.Expectation.Error)] 62 | } 63 | 64 | if diff := cmp.Diff(test.Expectation, act); diff != "" { 65 | t.Errorf("NewFilesystemCache() mismatch (-want +got):\n%s", diff) 66 | } 67 | 68 | if test.Expectation.Error == "" { 69 | // Cleanup created directory 70 | os.RemoveAll(test.Location) 71 | } 72 | }) 73 | } 74 | } 75 | 76 | func TestLocation(t *testing.T) { 77 | t.Parallel() 78 | 79 | type Expectation struct { 80 | Path string 81 | Exists bool 82 | Error string 83 | } 84 | 85 | tmpDir := t.TempDir() 86 | 87 | // Create test files 88 | err := os.WriteFile(filepath.Join(tmpDir, "test1.tar.gz"), []byte("test"), 0644) 89 | if err != nil { 90 | t.Fatal(err) 91 | } 92 | err = os.WriteFile(filepath.Join(tmpDir, "test2.tar"), []byte("test"), 0644) 93 | if err != nil { 94 | t.Fatal(err) 95 | } 96 | 97 | tests := []struct { 98 | Name string 99 | Cache *FilesystemCache 100 | Package cache.Package 101 | Expectation Expectation 102 | }{ 103 | { 104 | Name: "package version error", 105 | Cache: &FilesystemCache{Origin: tmpDir}, 106 | Package: mockPackage{ 107 | version: "", 108 | err: os.ErrNotExist, 109 | }, 110 | Expectation: Expectation{ 111 | Path: "", 112 | Exists: false, 113 | }, 114 | }, 115 | { 116 | Name: "existing tar.gz file", 117 | Cache: &FilesystemCache{Origin: tmpDir}, 118 | Package: mockPackage{ 119 | version: "test1", 120 | err: nil, 121 | }, 122 | Expectation: Expectation{ 123 | Path: filepath.Join(tmpDir, "test1.tar.gz"), 124 | Exists: true, 125 | }, 126 | }, 127 | { 128 | Name: "existing tar file", 129 | Cache: &FilesystemCache{Origin: tmpDir}, 130 | Package: mockPackage{ 131 | version: "test2", 132 | err: nil, 133 | }, 134 | Expectation: Expectation{ 135 | Path: filepath.Join(tmpDir, "test2.tar"), 136 | Exists: true, 137 | }, 138 | }, 139 | { 140 | Name: "non-existing file", 141 | Cache: &FilesystemCache{Origin: tmpDir}, 142 | Package: mockPackage{ 143 | version: "nonexistent", 144 | err: nil, 145 | }, 146 | Expectation: Expectation{ 147 | Path: filepath.Join(tmpDir, "nonexistent.tar"), 148 | Exists: false, 149 | }, 150 | }, 151 | } 152 | 153 | for _, test := range tests { 154 | test := test 155 | t.Run(test.Name, func(t *testing.T) { 156 | t.Parallel() 157 | var act Expectation 158 | 159 | path, exists := test.Cache.Location(test.Package) 160 | act.Path = path 161 | act.Exists = exists 162 | 163 | if diff := cmp.Diff(test.Expectation, act); diff != "" { 164 | t.Errorf("Location() mismatch (-want +got):\n%s", diff) 165 | } 166 | }) 167 | } 168 | } 169 | 170 | func TestFileExists(t *testing.T) { 171 | t.Parallel() 172 | 173 | tmpDir := t.TempDir() 174 | 175 | // Create a test file 176 | testFile := filepath.Join(tmpDir, "test.txt") 177 | err := os.WriteFile(testFile, []byte("test"), 0644) 178 | if err != nil { 179 | t.Fatal(err) 180 | } 181 | 182 | // Create a test directory 183 | testDir := filepath.Join(tmpDir, "testdir") 184 | err = os.Mkdir(testDir, 0755) 185 | if err != nil { 186 | t.Fatal(err) 187 | } 188 | 189 | tests := []struct { 190 | Name string 191 | Path string 192 | Expectation bool 193 | }{ 194 | { 195 | Name: "existing file", 196 | Path: testFile, 197 | Expectation: true, 198 | }, 199 | { 200 | Name: "directory", 201 | Path: testDir, 202 | Expectation: false, 203 | }, 204 | { 205 | Name: "non-existing file", 206 | Path: filepath.Join(tmpDir, "nonexistent.txt"), 207 | Expectation: false, 208 | }, 209 | } 210 | 211 | for _, test := range tests { 212 | test := test 213 | t.Run(test.Name, func(t *testing.T) { 214 | t.Parallel() 215 | 216 | result := fileExists(test.Path) 217 | if result != test.Expectation { 218 | t.Errorf("fileExists() = %v, want %v", result, test.Expectation) 219 | } 220 | }) 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /pkg/leeway/cache/remote/gsutil.go: -------------------------------------------------------------------------------- 1 | package remote 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "strings" 12 | "syscall" 13 | 14 | log "github.com/sirupsen/logrus" 15 | 16 | "github.com/gitpod-io/leeway/pkg/leeway/cache" 17 | ) 18 | 19 | // GSUtilCache uses the gsutil command to implement a remote cache 20 | type GSUtilCache struct { 21 | BucketName string 22 | } 23 | 24 | // NewGSUtilCache creates a new GSUtil cache implementation 25 | func NewGSUtilCache(cfg *cache.RemoteConfig) *GSUtilCache { 26 | return &GSUtilCache{ 27 | BucketName: cfg.BucketName, 28 | } 29 | } 30 | 31 | // ExistingPackages returns existing cached build artifacts in the remote cache 32 | func (rs *GSUtilCache) ExistingPackages(ctx context.Context, pkgs []cache.Package) (map[cache.Package]struct{}, error) { 33 | fmt.Printf("☁️ checking remote cache for past build artifacts for %d packages\n", len(pkgs)) 34 | 35 | // Map to store both .tar.gz and .tar URLs for each package 36 | type urlPair struct { 37 | gzURL string 38 | tarURL string 39 | } 40 | packageToURLMap := make(map[cache.Package]urlPair) 41 | 42 | // Create a list of all possible URLs 43 | var urls []string 44 | for _, p := range pkgs { 45 | version, err := p.Version() 46 | if err != nil { 47 | log.WithField("package", p.FullName()).Debug("Failed to get version for package. Will not check remote cache for package.") 48 | continue 49 | } 50 | 51 | pair := urlPair{ 52 | gzURL: fmt.Sprintf("gs://%s/%s.tar.gz", rs.BucketName, version), 53 | tarURL: fmt.Sprintf("gs://%s/%s.tar", rs.BucketName, version), 54 | } 55 | packageToURLMap[p] = pair 56 | urls = append(urls, pair.gzURL, pair.tarURL) 57 | } 58 | 59 | if len(urls) == 0 { 60 | return map[cache.Package]struct{}{}, nil 61 | } 62 | 63 | log.Debugf("Checking if %d packages exist in the remote cache using gsutil", len(urls)) 64 | args := append([]string{"stat"}, urls...) 65 | cmd := exec.Command("gsutil", args...) 66 | 67 | var stdoutBuffer, stderrBuffer strings.Builder 68 | cmd.Stdout = &stdoutBuffer 69 | cmd.Stderr = &stderrBuffer 70 | 71 | err := cmd.Run() 72 | if err != nil && (!strings.Contains(stderrBuffer.String(), "No URLs matched")) { 73 | log.Debugf("gsutil stat returned non-zero exit code: [%v], stderr: [%v]", err, stderrBuffer.String()) 74 | return map[cache.Package]struct{}{}, nil 75 | } 76 | 77 | existingURLs := parseGSUtilStatOutput(strings.NewReader(stdoutBuffer.String())) 78 | existingPackages := make(map[cache.Package]struct{}) 79 | 80 | for p, urls := range packageToURLMap { 81 | if _, exists := existingURLs[urls.gzURL]; exists { 82 | existingPackages[p] = struct{}{} 83 | continue 84 | } 85 | if _, exists := existingURLs[urls.tarURL]; exists { 86 | existingPackages[p] = struct{}{} 87 | } 88 | } 89 | 90 | return existingPackages, nil 91 | } 92 | 93 | // Download makes a best-effort attempt at downloading previously cached build artifacts 94 | func (rs *GSUtilCache) Download(ctx context.Context, dst cache.LocalCache, pkgs []cache.Package) error { 95 | fmt.Printf("☁️ downloading %d cached build artifacts\n", len(pkgs)) 96 | var ( 97 | files []string 98 | dest string 99 | ) 100 | type urlPair struct { 101 | gzURL string 102 | tarURL string 103 | } 104 | 105 | // Create a list of all possible URLs 106 | var urls []string 107 | packageToURLMap := make(map[cache.Package]urlPair) 108 | for _, pkg := range pkgs { 109 | fn, exists := dst.Location(pkg) 110 | if exists { 111 | continue 112 | } 113 | version, err := pkg.Version() 114 | if err != nil { 115 | log.WithError(err).WithField("package", pkg.FullName()).Warn("Failed to get version for package, skipping") 116 | continue 117 | } 118 | if dest == "" { 119 | dest = filepath.Dir(fn) 120 | } else if dest != filepath.Dir(fn) { 121 | return fmt.Errorf("gsutil only supports one target folder, not %s and %s", dest, filepath.Dir(fn)) 122 | } 123 | 124 | pair := urlPair{ 125 | gzURL: fmt.Sprintf("gs://%s/%s.tar.gz", rs.BucketName, version), 126 | tarURL: fmt.Sprintf("gs://%s/%s.tar", rs.BucketName, version), 127 | } 128 | packageToURLMap[pkg] = pair 129 | urls = append(urls, pair.gzURL, pair.tarURL) 130 | } 131 | if len(urls) == 0 { 132 | return nil 133 | } 134 | 135 | args := append([]string{"stat"}, urls...) 136 | cmd := exec.Command("gsutil", args...) 137 | 138 | var stdoutBuffer, stderrBuffer strings.Builder 139 | cmd.Stdout = &stdoutBuffer 140 | cmd.Stderr = &stderrBuffer 141 | 142 | err := cmd.Run() 143 | if err != nil && (!strings.Contains(stderrBuffer.String(), "No URLs matched")) { 144 | log.Debugf("gsutil stat returned non-zero exit code: [%v], stderr: [%v]", err, stderrBuffer.String()) 145 | return fmt.Errorf("failed to check if files exist in remote cache: %w", err) 146 | } 147 | 148 | existingURLs := parseGSUtilStatOutput(strings.NewReader(stdoutBuffer.String())) 149 | for _, urls := range packageToURLMap { 150 | if _, exists := existingURLs[urls.gzURL]; exists { 151 | files = append(files, urls.gzURL) 152 | continue 153 | } 154 | if _, exists := existingURLs[urls.tarURL]; exists { 155 | files = append(files, urls.tarURL) 156 | } 157 | } 158 | 159 | return gsutilTransfer(dest, files) 160 | } 161 | 162 | // Upload makes a best effort to upload the build artifacts to a remote cache 163 | func (rs *GSUtilCache) Upload(ctx context.Context, src cache.LocalCache, pkgs []cache.Package) error { 164 | fmt.Printf("☁️ uploading build artifacts to remote cache\n") 165 | var files []string 166 | for _, pkg := range pkgs { 167 | file, exists := src.Location(pkg) 168 | if !exists { 169 | continue 170 | } 171 | files = append(files, file) 172 | } 173 | return gsutilTransfer(fmt.Sprintf("gs://%s", rs.BucketName), files) 174 | } 175 | 176 | func parseGSUtilStatOutput(reader io.Reader) map[string]struct{} { 177 | exists := make(map[string]struct{}) 178 | scanner := bufio.NewScanner(reader) 179 | for scanner.Scan() { 180 | line := scanner.Text() 181 | if strings.HasPrefix(line, "gs://") { 182 | url := strings.TrimSuffix(line, ":") 183 | exists[url] = struct{}{} 184 | continue 185 | } 186 | } 187 | return exists 188 | } 189 | 190 | func gsutilTransfer(target string, files []string) error { 191 | log.WithField("target", target).WithField("files", files).Debug("Transferring files using gsutil") 192 | 193 | cmd := exec.Command("gsutil", "-m", "cp", "-I", target) 194 | cmd.Stdout = os.Stdout 195 | stdin, err := cmd.StdinPipe() 196 | if err != nil { 197 | return err 198 | } 199 | err = cmd.Start() 200 | if err != nil { 201 | return err 202 | } 203 | 204 | for _, fn := range files { 205 | _, err = fmt.Fprintln(stdin, fn) 206 | if err != nil { 207 | return err 208 | } 209 | } 210 | err = stdin.Close() 211 | if err != nil { 212 | return err 213 | } 214 | 215 | err = cmd.Wait() 216 | if err != nil { 217 | if exiterr, ok := err.(*exec.ExitError); ok { 218 | if _, ok := exiterr.Sys().(syscall.WaitStatus); ok { 219 | // we just swallow non-zero exit codes here as remote caching is best effort 220 | return nil 221 | } 222 | } 223 | 224 | return err 225 | } 226 | return nil 227 | } 228 | -------------------------------------------------------------------------------- /pkg/leeway/cache/remote/no_cache.go: -------------------------------------------------------------------------------- 1 | package remote 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/gitpod-io/leeway/pkg/leeway/cache" 7 | ) 8 | 9 | // NoRemoteCache implements the default no-remote cache behavior 10 | type NoRemoteCache struct{} 11 | 12 | // NewNoRemoteCache creates a new NoRemoteCache instance 13 | func NewNoRemoteCache() *NoRemoteCache { 14 | return &NoRemoteCache{} 15 | } 16 | 17 | // ExistingPackages returns existing cached build artifacts in the remote cache 18 | func (NoRemoteCache) ExistingPackages(ctx context.Context, pkgs []cache.Package) (map[cache.Package]struct{}, error) { 19 | return map[cache.Package]struct{}{}, nil 20 | } 21 | 22 | // Download makes a best-effort attempt at downloading previously cached build artifacts 23 | func (NoRemoteCache) Download(ctx context.Context, dst cache.LocalCache, pkgs []cache.Package) error { 24 | return nil 25 | } 26 | 27 | // Upload makes a best effort to upload the build artifacts to a remote cache 28 | func (NoRemoteCache) Upload(ctx context.Context, src cache.LocalCache, pkgs []cache.Package) error { 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /pkg/leeway/cache/remote/no_cache_test.go: -------------------------------------------------------------------------------- 1 | package remote 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/gitpod-io/leeway/pkg/leeway/cache" 8 | ) 9 | 10 | type mockPackage struct { 11 | version string 12 | err error 13 | } 14 | 15 | func (m mockPackage) Version() (string, error) { 16 | return m.version, m.err 17 | } 18 | 19 | func (m mockPackage) FullName() string { 20 | return "mock-package" 21 | } 22 | 23 | type mockLocalCache struct { 24 | locations map[string]string 25 | } 26 | 27 | func (m *mockLocalCache) Location(pkg cache.Package) (path string, exists bool) { 28 | version, err := pkg.Version() 29 | if err != nil { 30 | return "", false 31 | } 32 | path, exists = m.locations[version] 33 | return 34 | } 35 | 36 | func TestNewNoRemoteCache(t *testing.T) { 37 | t.Parallel() 38 | 39 | cache := NewNoRemoteCache() 40 | if cache == nil { 41 | t.Error("NewNoRemoteCache() returned nil") 42 | } 43 | } 44 | 45 | func TestExistingPackages(t *testing.T) { 46 | t.Parallel() 47 | 48 | noCache := NewNoRemoteCache() 49 | packages := []mockPackage{ 50 | {version: "test1", err: nil}, 51 | {version: "test2", err: nil}, 52 | } 53 | 54 | var pkgs []cache.Package 55 | for i := range packages { 56 | pkgs = append(pkgs, packages[i]) 57 | } 58 | 59 | result, err := noCache.ExistingPackages(context.Background(), pkgs) 60 | if err != nil { 61 | t.Errorf("ExistingPackages() error = %v", err) 62 | } 63 | 64 | if len(result) != 0 { 65 | t.Errorf("ExistingPackages() = %v, want empty map", result) 66 | } 67 | } 68 | 69 | func TestDownload(t *testing.T) { 70 | t.Parallel() 71 | 72 | noCache := NewNoRemoteCache() 73 | localCache := &mockLocalCache{ 74 | locations: map[string]string{ 75 | "test1": "/path/to/test1.tar.gz", 76 | "test2": "/path/to/test2.tar", 77 | }, 78 | } 79 | packages := []mockPackage{ 80 | {version: "test1", err: nil}, 81 | {version: "test2", err: nil}, 82 | } 83 | 84 | var pkgs []cache.Package 85 | for i := range packages { 86 | pkgs = append(pkgs, packages[i]) 87 | } 88 | 89 | err := noCache.Download(context.Background(), localCache, pkgs) 90 | if err != nil { 91 | t.Errorf("Download() error = %v", err) 92 | } 93 | } 94 | 95 | func TestUpload(t *testing.T) { 96 | t.Parallel() 97 | 98 | noCache := NewNoRemoteCache() 99 | localCache := &mockLocalCache{ 100 | locations: map[string]string{ 101 | "test1": "/path/to/test1.tar.gz", 102 | "test2": "/path/to/test2.tar", 103 | }, 104 | } 105 | packages := []mockPackage{ 106 | {version: "test1", err: nil}, 107 | {version: "test2", err: nil}, 108 | } 109 | 110 | var pkgs []cache.Package 111 | for i := range packages { 112 | pkgs = append(pkgs, packages[i]) 113 | } 114 | 115 | err := noCache.Upload(context.Background(), localCache, pkgs) 116 | if err != nil { 117 | t.Errorf("Upload() error = %v", err) 118 | } 119 | } 120 | 121 | func TestNoRemoteCacheImplementsInterface(t *testing.T) { 122 | t.Parallel() 123 | 124 | var _ cache.RemoteCache = (*NoRemoteCache)(nil) 125 | } 126 | 127 | // Ensure mockPackage implements cache.Package interface 128 | var _ cache.Package = mockPackage{} 129 | 130 | // Ensure mockLocalCache implements cache.LocalCache interface 131 | var _ cache.LocalCache = (*mockLocalCache)(nil) 132 | -------------------------------------------------------------------------------- /pkg/leeway/cache/remote/s3_download_test.go: -------------------------------------------------------------------------------- 1 | package remote 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "testing" 10 | "time" 11 | 12 | "github.com/gitpod-io/leeway/pkg/leeway/cache" 13 | "github.com/gitpod-io/leeway/pkg/leeway/cache/local" 14 | "github.com/google/go-cmp/cmp" 15 | ) 16 | 17 | // s3TestPackage for testing S3 download functionality 18 | type s3TestPackage struct { 19 | versionStr string 20 | fullName string 21 | versionErr error 22 | } 23 | 24 | func (m s3TestPackage) Version() (string, error) { 25 | return m.versionStr, m.versionErr 26 | } 27 | 28 | func (m s3TestPackage) FullName() string { 29 | return m.fullName 30 | } 31 | 32 | // mockS3Storage implements a simple ObjectStorage for testing 33 | type mockS3Storage struct { 34 | objects map[string][]byte 35 | failDownload bool 36 | downloadDelay time.Duration 37 | } 38 | 39 | func (m *mockS3Storage) HasObject(ctx context.Context, key string) (bool, error) { 40 | _, exists := m.objects[key] 41 | return exists, nil 42 | } 43 | 44 | func (m *mockS3Storage) GetObject(ctx context.Context, key string, dest string) (int64, error) { 45 | if m.failDownload { 46 | return 0, errors.New("simulated download failure") 47 | } 48 | 49 | if m.downloadDelay > 0 { 50 | time.Sleep(m.downloadDelay) 51 | } 52 | 53 | data, exists := m.objects[key] 54 | if !exists { 55 | return 0, errors.New("NotFound: object does not exist") 56 | } 57 | 58 | if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { 59 | return 0, err 60 | } 61 | 62 | if err := os.WriteFile(dest, data, 0644); err != nil { 63 | return 0, err 64 | } 65 | 66 | return int64(len(data)), nil 67 | } 68 | 69 | func (m *mockS3Storage) UploadObject(ctx context.Context, key string, src string) error { 70 | data, err := os.ReadFile(src) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | m.objects[key] = data 76 | return nil 77 | } 78 | 79 | func (m *mockS3Storage) ListObjects(ctx context.Context, prefix string) ([]string, error) { 80 | var result []string 81 | for key := range m.objects { 82 | if len(prefix) == 0 || key[:len(prefix)] == prefix { 83 | result = append(result, key) 84 | } 85 | } 86 | return result, nil 87 | } 88 | 89 | func TestS3CacheDownload(t *testing.T) { 90 | tmpDir := t.TempDir() 91 | 92 | localCache, err := local.NewFilesystemCache(tmpDir) 93 | if err != nil { 94 | t.Fatalf("failed to create local cache: %v", err) 95 | } 96 | 97 | tests := []struct { 98 | name string 99 | packages []cache.Package 100 | mockObjects map[string][]byte 101 | failDownload bool 102 | downloadDelay time.Duration 103 | wantDownloaded map[string]bool 104 | wantErrorCount int 105 | }{ 106 | { 107 | name: "successful download", 108 | packages: []cache.Package{ 109 | s3TestPackage{versionStr: "v1", fullName: "pkg1"}, 110 | }, 111 | mockObjects: map[string][]byte{ 112 | "v1.tar.gz": []byte("test data"), 113 | }, 114 | wantDownloaded: map[string]bool{ 115 | "v1.tar.gz": true, 116 | }, 117 | }, 118 | { 119 | name: "fallback to tar", 120 | packages: []cache.Package{ 121 | s3TestPackage{versionStr: "v2", fullName: "pkg2"}, 122 | }, 123 | mockObjects: map[string][]byte{ 124 | "v2.tar": []byte("test data for tar"), 125 | }, 126 | wantDownloaded: map[string]bool{ 127 | "v2.tar": true, 128 | }, 129 | }, 130 | { 131 | name: "package not found", 132 | packages: []cache.Package{ 133 | s3TestPackage{versionStr: "v3", fullName: "pkg3"}, 134 | }, 135 | mockObjects: map[string][]byte{}, 136 | wantDownloaded: map[string]bool{}, 137 | }, 138 | { 139 | name: "download failure with retry success", 140 | packages: []cache.Package{ 141 | s3TestPackage{versionStr: "v4", fullName: "pkg4"}, 142 | }, 143 | mockObjects: map[string][]byte{ 144 | "v4.tar.gz": []byte("test data"), 145 | }, 146 | downloadDelay: 10 * time.Millisecond, // Simulate network latency 147 | wantDownloaded: map[string]bool{ 148 | "v4.tar.gz": true, 149 | }, 150 | }, 151 | { 152 | name: "package version error", 153 | packages: []cache.Package{ 154 | s3TestPackage{versionStr: "", fullName: "pkg5", versionErr: errors.New("version error")}, 155 | }, 156 | mockObjects: map[string][]byte{}, 157 | wantDownloaded: map[string]bool{}, 158 | }, 159 | } 160 | 161 | for _, tt := range tests { 162 | t.Run(tt.name, func(t *testing.T) { 163 | // Clean up the temp directory 164 | files, _ := filepath.Glob(filepath.Join(tmpDir, "*")) 165 | for _, f := range files { 166 | os.Remove(f) 167 | } 168 | 169 | mockStorage := &mockS3Storage{ 170 | objects: tt.mockObjects, 171 | failDownload: tt.failDownload, 172 | downloadDelay: tt.downloadDelay, 173 | } 174 | 175 | s3Cache := &S3Cache{ 176 | storage: mockStorage, 177 | workerCount: 1, 178 | } 179 | 180 | err := s3Cache.Download(context.Background(), localCache, tt.packages) 181 | 182 | // We always return nil from Download now to continue with local builds 183 | if err != nil { 184 | t.Errorf("expected no error but got %v", err) 185 | } 186 | 187 | // Check if expected files were downloaded 188 | for pkg, expectedDownload := range tt.wantDownloaded { 189 | // We need to look for the file based on the version as that's how the local cache stores it 190 | // Extract the version from the package name (v1, v2, etc.) 191 | version := strings.TrimSuffix(strings.TrimSuffix(pkg, ".tar.gz"), ".tar") 192 | path := filepath.Join(tmpDir, pkg) 193 | _, err := os.Stat(path) 194 | fileExists := err == nil 195 | 196 | if expectedDownload && !fileExists { 197 | // Check if any file with the version prefix exists 198 | matches, _ := filepath.Glob(filepath.Join(tmpDir, version+"*")) 199 | if len(matches) == 0 { 200 | t.Errorf("expected a file for package %s to be downloaded but none was found", pkg) 201 | } 202 | } else if !expectedDownload && fileExists { 203 | t.Errorf("didn't expect file %s to be downloaded but it was", pkg) 204 | } 205 | 206 | if fileExists { 207 | data, _ := os.ReadFile(path) 208 | if !cmp.Equal(data, mockStorage.objects[pkg]) { 209 | t.Errorf("file contents mismatch for %s", pkg) 210 | } 211 | } 212 | } 213 | }) 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /pkg/leeway/cache/testing/mock.go: -------------------------------------------------------------------------------- 1 | package testing 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path/filepath" 7 | "sync" 8 | 9 | "github.com/gitpod-io/leeway/pkg/leeway/cache" 10 | ) 11 | 12 | // MockLocalCache implements a in-memory LocalCache for testing 13 | type MockLocalCache struct { 14 | files map[string][]byte 15 | mu sync.RWMutex 16 | } 17 | 18 | // NewMockLocalCache creates a new mock local cache 19 | func NewMockLocalCache() *MockLocalCache { 20 | return &MockLocalCache{ 21 | files: make(map[string][]byte), 22 | } 23 | } 24 | 25 | // Location implements LocalCache 26 | func (m *MockLocalCache) Location(pkg cache.Package) (path string, exists bool) { 27 | version, err := pkg.Version() 28 | if err != nil { 29 | return "", false 30 | } 31 | 32 | m.mu.RLock() 33 | defer m.mu.RUnlock() 34 | 35 | // Check for .tar.gz first 36 | gzPath := fmt.Sprintf("%s.tar.gz", version) 37 | if _, ok := m.files[gzPath]; ok { 38 | return gzPath, true 39 | } 40 | 41 | // Fall back to .tar 42 | tarPath := fmt.Sprintf("%s.tar", version) 43 | _, exists = m.files[tarPath] 44 | return tarPath, exists 45 | } 46 | 47 | // AddFile adds a file to the mock cache 48 | func (m *MockLocalCache) AddFile(path string, content []byte) { 49 | m.mu.Lock() 50 | defer m.mu.Unlock() 51 | m.files[path] = content 52 | } 53 | 54 | // MockObjectStorage implements ObjectStorage interface for testing 55 | type MockObjectStorage struct { 56 | objects map[string][]byte 57 | mu sync.RWMutex 58 | } 59 | 60 | // NewMockObjectStorage creates a new mock object storage 61 | func NewMockObjectStorage() *MockObjectStorage { 62 | return &MockObjectStorage{ 63 | objects: make(map[string][]byte), 64 | } 65 | } 66 | 67 | // HasObject implements ObjectStorage 68 | func (m *MockObjectStorage) HasObject(ctx context.Context, key string) (bool, error) { 69 | m.mu.RLock() 70 | defer m.mu.RUnlock() 71 | _, exists := m.objects[key] 72 | return exists, nil 73 | } 74 | 75 | // GetObject implements ObjectStorage 76 | func (m *MockObjectStorage) GetObject(ctx context.Context, key string, dest string) (int64, error) { 77 | m.mu.RLock() 78 | content, exists := m.objects[key] 79 | m.mu.RUnlock() 80 | 81 | if !exists { 82 | return 0, fmt.Errorf("object not found: %s", key) 83 | } 84 | 85 | return int64(len(content)), nil 86 | } 87 | 88 | // UploadObject implements ObjectStorage 89 | func (m *MockObjectStorage) UploadObject(ctx context.Context, key string, src string) error { 90 | m.mu.Lock() 91 | defer m.mu.Unlock() 92 | m.objects[key] = []byte(fmt.Sprintf("mock content for %s", key)) 93 | return nil 94 | } 95 | 96 | // ListObjects implements ObjectStorage 97 | func (m *MockObjectStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { 98 | m.mu.RLock() 99 | defer m.mu.RUnlock() 100 | 101 | var result []string 102 | for key := range m.objects { 103 | if matched, _ := filepath.Match(prefix+"*", key); matched { 104 | result = append(result, key) 105 | } 106 | } 107 | return result, nil 108 | } 109 | 110 | // AddObject adds an object to the mock storage 111 | func (m *MockObjectStorage) AddObject(key string, content []byte) { 112 | m.mu.Lock() 113 | defer m.mu.Unlock() 114 | m.objects[key] = content 115 | } 116 | -------------------------------------------------------------------------------- /pkg/leeway/cache/types.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | // Package represents a build package that can be cached 8 | type Package interface { 9 | // Version returns a unique identifier for the package 10 | Version() (string, error) 11 | // FullName returns the full name of the package 12 | FullName() string 13 | } 14 | 15 | // LocalCache provides filesystem locations for package build artifacts 16 | type LocalCache interface { 17 | // Location returns the absolute filesystem path for a package build artifact 18 | Location(pkg Package) (path string, exists bool) 19 | } 20 | 21 | // RemoteCache can download and upload build artifacts into a local cache 22 | type RemoteCache interface { 23 | // ExistingPackages returns existing cached build artifacts in the remote cache 24 | ExistingPackages(ctx context.Context, pkgs []Package) (map[Package]struct{}, error) 25 | 26 | // Download makes a best-effort attempt at downloading previously cached build artifacts 27 | // A cache miss does not constitute an error 28 | Download(ctx context.Context, dst LocalCache, pkgs []Package) error 29 | 30 | // Upload makes a best effort to upload the build artifacts to a remote cache 31 | Upload(ctx context.Context, src LocalCache, pkgs []Package) error 32 | } 33 | 34 | // ObjectStorage represents a generic object storage interface 35 | // This allows us to abstract S3, GCS, or other storage backends 36 | type ObjectStorage interface { 37 | // HasObject checks if an object exists 38 | HasObject(ctx context.Context, key string) (bool, error) 39 | 40 | // GetObject downloads an object to a local file 41 | GetObject(ctx context.Context, key string, dest string) (int64, error) 42 | 43 | // UploadObject uploads a local file to remote storage 44 | UploadObject(ctx context.Context, key string, src string) error 45 | 46 | // ListObjects lists objects with the given prefix 47 | ListObjects(ctx context.Context, prefix string) ([]string, error) 48 | } 49 | 50 | // Config holds configuration for cache implementations 51 | type Config struct { 52 | // Location is the base path for local cache 53 | Location string 54 | 55 | // RemoteConfig holds remote cache specific configuration 56 | RemoteConfig RemoteConfig 57 | } 58 | 59 | // RemoteConfig holds configuration for remote cache implementations 60 | type RemoteConfig struct { 61 | // BucketName for object storage 62 | BucketName string 63 | 64 | // Region for services that require it (e.g. S3) 65 | Region string 66 | 67 | // Endpoint for the remote service 68 | Endpoint string 69 | } 70 | -------------------------------------------------------------------------------- /pkg/leeway/export.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "strings" 9 | 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // CopyWorkspace copies all folders/files from a workspace to a destination. 14 | // If strict is true we'll only copy the files that leeway actully knows are source files. 15 | // Otherwise we'll copy all files that are not excluded by the variant. 16 | func CopyWorkspace(dst string, workspace *Workspace, strict bool) error { 17 | out, err := exec.Command("cp", "-R", workspace.Origin, dst).CombinedOutput() 18 | if err != nil { 19 | return fmt.Errorf("%w: %s", err, string(out)) 20 | } 21 | 22 | return DeleteNonWorkspaceFiles(dst, workspace, strict) 23 | } 24 | 25 | // DeleteNonWorkspaceFiles removes all files that do not belong to a workspace. 26 | // If strict is true this function deletes all files that are not listed as source in a package. 27 | // If strict is fales this function deletes files excluded by a variant. 28 | func DeleteNonWorkspaceFiles(dst string, workspace *Workspace, strict bool) (err error) { 29 | var ( 30 | excl = make(map[string]struct{}) 31 | incl = make(map[string]struct{}) 32 | ) 33 | if strict { 34 | for _, pkg := range workspace.Packages { 35 | for _, s := range pkg.Sources { 36 | rels := strings.TrimPrefix(s, workspace.Origin) 37 | incl[rels] = struct{}{} 38 | 39 | // package sources are files only - we need to include their parent directories as well 40 | for p := filepath.Dir(rels); p != "/"; p = filepath.Dir(p) { 41 | incl[p] = struct{}{} 42 | } 43 | } 44 | } 45 | } else { 46 | err = filepath.Walk(workspace.Origin, func(path string, info os.FileInfo, err error) error { 47 | s := strings.TrimPrefix(path, workspace.Origin) 48 | incl[s] = struct{}{} 49 | return nil 50 | }) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | if workspace.SelectedVariant != nil { 56 | vinc, vexc, err := workspace.SelectedVariant.ResolveSources(workspace, dst) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | for _, p := range vinc { 62 | incl[strings.TrimPrefix(p, dst)] = struct{}{} 63 | } 64 | for _, p := range vexc { 65 | excl[strings.TrimPrefix(p, dst)] = struct{}{} 66 | } 67 | } 68 | } 69 | 70 | // keep if incl and not excl 71 | return filepath.Walk(dst, func(path string, info os.FileInfo, err error) error { 72 | if path == dst { 73 | return nil 74 | } 75 | 76 | s := strings.TrimPrefix(path, dst) 77 | _, inc := incl[s] 78 | _, exc := excl[s] 79 | lg := log.WithField("inc", inc).WithField("exc", exc).WithField("s", s).WithField("dst", dst) 80 | if inc && !exc { 81 | lg.Debug("not deleting file") 82 | return nil 83 | } 84 | lg.Debug("deleting file") 85 | 86 | return os.RemoveAll(path) 87 | }) 88 | } 89 | -------------------------------------------------------------------------------- /pkg/leeway/fixtures/load-workspace.yaml: -------------------------------------------------------------------------------- 1 | workspace: 2 | environmentManifest: 3 | - name: "foobar" 4 | command: ["echo", "foobar"] 5 | defaultArgs: 6 | message: "hello root" 7 | 8 | components: 9 | - location: wsa/pkg0 10 | packages: 11 | - name: app 12 | type: generic 13 | config: 14 | commands: 15 | - ["echo"] 16 | - location: wsa/pkg1 17 | packages: 18 | - name: app 19 | type: generic 20 | config: 21 | commands: 22 | - ["echo"] 23 | scripts: 24 | - name: echo 25 | script: | 26 | echo ${message} 27 | - location: wsa 28 | packages: 29 | - name: app 30 | type: generic 31 | config: 32 | commands: 33 | - ["echo", "${message}"] 34 | scripts: 35 | - name: echo 36 | script: | 37 | echo ${message} 38 | - location: deeper/pkg0 39 | packages: 40 | - name: app 41 | type: generic 42 | deps: 43 | - wsa/pkg0:app 44 | - wsa/pkg1:app 45 | - wsa:app 46 | config: 47 | commands: 48 | - ["echo"] 49 | -------------------------------------------------------------------------------- /pkg/leeway/format.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "io" 5 | "sort" 6 | 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | // FormatBUILDyaml formats a component's build.yaml file 11 | func FormatBUILDyaml(out io.Writer, in io.Reader, fixIssues bool) error { 12 | var n yaml.Node 13 | err := yaml.NewDecoder(in).Decode(&n) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | sortPackageDeps(&n) 19 | // if fixIssues { 20 | // right now we have no automatic issue fixes - if this changes, add them here 21 | // } 22 | 23 | enc := yaml.NewEncoder(out) 24 | enc.SetIndent(2) 25 | return enc.Encode(&n) 26 | } 27 | 28 | func sortPackageDeps(n *yaml.Node) { 29 | if n == nil || len(n.Content) < 1 { 30 | return 31 | } 32 | 33 | // Get the root mapping node 34 | root := n.Content[0] 35 | if root.Kind != yaml.MappingNode { 36 | return 37 | } 38 | 39 | // Find the packages section 40 | for i := 0; i < len(root.Content); i += 2 { 41 | if root.Content[i].Value != "packages" { 42 | continue 43 | } 44 | 45 | // Get the packages mapping 46 | packagesNode := root.Content[i+1] 47 | if packagesNode.Kind != yaml.MappingNode { 48 | return 49 | } 50 | 51 | // Iterate through each package 52 | for j := 0; j < len(packagesNode.Content); j += 2 { 53 | pkg := packagesNode.Content[j+1] 54 | if pkg.Kind != yaml.MappingNode { 55 | continue 56 | } 57 | 58 | // Find deps in the package 59 | for k := 0; k < len(pkg.Content); k += 2 { 60 | if pkg.Content[k].Value != "deps" { 61 | continue 62 | } 63 | 64 | // Get the deps sequence 65 | depsNode := pkg.Content[k+1] 66 | if depsNode.Kind != yaml.SequenceNode { 67 | continue 68 | } 69 | 70 | // Sort the dependencies 71 | sort.Slice(depsNode.Content, func(i, j int) bool { 72 | return depsNode.Content[i].Value < depsNode.Content[j].Value 73 | }) 74 | } 75 | } 76 | break 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /pkg/leeway/format_test.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | func TestSortPackageDeps(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | input string 14 | expected string 15 | }{ 16 | { 17 | name: "empty yaml", 18 | input: "", 19 | expected: "", 20 | }, 21 | { 22 | name: "no packages", 23 | input: ` 24 | other: 25 | field: value`, 26 | expected: ` 27 | other: 28 | field: value`, 29 | }, 30 | { 31 | name: "packages without deps", 32 | input: ` 33 | packages: 34 | some-package: 35 | name: test 36 | type: go`, 37 | expected: ` 38 | packages: 39 | some-package: 40 | name: test 41 | type: go`, 42 | }, 43 | { 44 | name: "packages with unsorted deps", 45 | input: ` 46 | packages: 47 | some-package: 48 | name: test 49 | type: go 50 | deps: 51 | - :package-c 52 | - :package-a 53 | - :package-b`, 54 | expected: ` 55 | packages: 56 | some-package: 57 | name: test 58 | type: go 59 | deps: 60 | - :package-a 61 | - :package-b 62 | - :package-c`, 63 | }, 64 | { 65 | name: "multiple packages with deps", 66 | input: ` 67 | packages: 68 | package1: 69 | deps: 70 | - :c 71 | - :a 72 | - :b 73 | package2: 74 | deps: 75 | - :z 76 | - :x 77 | - :y`, 78 | expected: ` 79 | packages: 80 | package1: 81 | deps: 82 | - :a 83 | - :b 84 | - :c 85 | package2: 86 | deps: 87 | - :x 88 | - :y 89 | - :z`, 90 | }, 91 | } 92 | 93 | for _, tc := range tests { 94 | t.Run(tc.name, func(t *testing.T) { 95 | // Parse input YAML 96 | var inputNode yaml.Node 97 | err := yaml.Unmarshal([]byte(tc.input), &inputNode) 98 | if tc.input != "" && err != nil { 99 | t.Fatalf("failed to unmarshal input YAML: %v", err) 100 | } 101 | 102 | // Sort dependencies 103 | sortPackageDeps(&inputNode) 104 | 105 | // Marshal back to string 106 | var output strings.Builder 107 | enc := yaml.NewEncoder(&output) 108 | enc.SetIndent(2) 109 | if tc.input != "" { 110 | if err := enc.Encode(&inputNode); err != nil { 111 | t.Fatalf("failed to marshal output YAML: %v", err) 112 | } 113 | } 114 | 115 | // Compare with expected 116 | expectedStr := strings.TrimSpace(tc.expected) 117 | actualStr := strings.TrimSpace(output.String()) 118 | if expectedStr != actualStr { 119 | t.Errorf("\nexpected:\n%s\n\nactual:\n%s", expectedStr, actualStr) 120 | } 121 | }) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /pkg/leeway/gitinfo.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "strings" 9 | 10 | log "github.com/sirupsen/logrus" 11 | "golang.org/x/xerrors" 12 | ) 13 | 14 | const ( 15 | gitDirName = ".git" 16 | gitStatusError = 128 17 | ) 18 | 19 | // GitError represents an error that occurred during a Git operation 20 | type GitError struct { 21 | Op string 22 | Err error 23 | } 24 | 25 | func (e *GitError) Error() string { 26 | return fmt.Sprintf("git operation %s failed: %v", e.Op, e.Err) 27 | } 28 | 29 | // GitInfo represents the state of a Git working copy including commit information, 30 | // origin URL, and dirty state tracking. 31 | type GitInfo struct { 32 | // WorkingCopyLoc is the absolute path to the Git working copy 33 | WorkingCopyLoc string 34 | // Commit is the current HEAD commit hash 35 | Commit string 36 | // Origin is the remote origin URL 37 | Origin string 38 | 39 | dirty bool 40 | dirtyFiles map[string]struct{} 41 | } 42 | 43 | // executeGitCommand is a helper function to execute Git commands and handle their output 44 | func executeGitCommand(dir string, args ...string) (string, error) { 45 | cmd := exec.Command("git", args...) 46 | cmd.Dir = dir 47 | out, err := cmd.CombinedOutput() 48 | if err != nil { 49 | return "", &GitError{ 50 | Op: strings.Join(args, " "), 51 | Err: err, 52 | } 53 | } 54 | return strings.TrimSpace(string(out)), nil 55 | } 56 | 57 | // GetGitInfo returns the git status required during a leeway build 58 | func GetGitInfo(loc string) (*GitInfo, error) { 59 | gitfc := filepath.Join(loc, gitDirName) 60 | stat, err := os.Stat(gitfc) 61 | if err != nil || !stat.IsDir() { 62 | return nil, nil 63 | } 64 | 65 | commit, err := executeGitCommand(loc, "rev-parse", "HEAD") 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | res := GitInfo{ 71 | WorkingCopyLoc: loc, 72 | Commit: commit, 73 | } 74 | 75 | origin, err := executeGitCommand(loc, "config", "--get", "remote.origin.url") 76 | if err == nil { 77 | res.Origin = origin 78 | } 79 | 80 | status, err := executeGitCommand(loc, "status", "--porcelain") 81 | if serr, ok := err.(*exec.ExitError); ok && serr.ExitCode() != gitStatusError { 82 | log.WithFields(log.Fields{ 83 | "exitCode": serr.ExitCode(), 84 | "workingCopy": loc, 85 | }).Debug("git status --porcelain exited with failed exit code. Working copy is dirty.") 86 | res.dirty = true 87 | } else if _, ok := err.(*exec.ExitError); !ok && err != nil { 88 | return nil, err 89 | } else if status != "" { 90 | log.WithFields(log.Fields{ 91 | "workingCopy": loc, 92 | "status": status, 93 | }).Debug("working copy is dirty") 94 | 95 | res.dirty = true 96 | res.dirtyFiles, err = parseGitStatus([]byte(status)) 97 | if err != nil { 98 | log.WithError(err).Warn("cannot parse git status: assuming all files are dirty") 99 | } 100 | } 101 | 102 | return &res, nil 103 | } 104 | 105 | // parseGitStatus parses the output of "git status --porcelain" 106 | func parseGitStatus(out []byte) (files map[string]struct{}, err error) { 107 | in := strings.TrimSpace(string(out)) 108 | if len(in) == 0 { 109 | // no files - nothing's dirty 110 | return nil, nil 111 | } 112 | 113 | lines := strings.Split(in, "\n") 114 | files = make(map[string]struct{}, len(lines)) 115 | for _, l := range lines { 116 | segs := strings.Fields(l) 117 | if len(segs) == 0 { 118 | continue 119 | } 120 | if len(segs) != 2 { 121 | return nil, xerrors.Errorf("cannot parse git status \"%s\": expected two segments, got %d", l, len(segs)) 122 | } 123 | files[segs[1]] = struct{}{} 124 | } 125 | return 126 | } 127 | 128 | // IsDirty returns whether the working copy has any modifications 129 | func (info *GitInfo) IsDirty() bool { 130 | return info.dirty 131 | } 132 | 133 | // HasDirtyFile checks if a specific file is dirty 134 | func (info *GitInfo) HasDirtyFile(file string) bool { 135 | if !info.dirty { 136 | return false 137 | } 138 | if len(info.dirtyFiles) == 0 { 139 | return true 140 | } 141 | 142 | file = strings.TrimPrefix(file, info.WorkingCopyLoc) 143 | file = strings.TrimPrefix(file, "/") 144 | _, isDirty := info.dirtyFiles[file] 145 | return isDirty 146 | } 147 | 148 | // DirtyFiles returns true if a single file of the file list is dirty 149 | func (info *GitInfo) DirtyFiles(files []string) bool { 150 | if !info.dirty { 151 | log.WithField("workingCopy", info.WorkingCopyLoc).Debug("building from a clean working copy") 152 | return false 153 | } 154 | if len(info.dirtyFiles) == 0 { 155 | log.WithField("workingCopy", info.WorkingCopyLoc).Debug("no records of dirty files - assuming dirty Git working copy") 156 | return true 157 | } 158 | for _, f := range files { 159 | if !strings.HasPrefix(f, info.WorkingCopyLoc) { 160 | log.WithFields(log.Fields{ 161 | "workingCopy": info.WorkingCopyLoc, 162 | "file": f, 163 | }).Debug("no records of this file - assuming it's dirty") 164 | return true 165 | } 166 | 167 | if info.HasDirtyFile(f) { 168 | log.WithFields(log.Fields{ 169 | "workingCopy": info.WorkingCopyLoc, 170 | "file": f, 171 | }).Debug("found dirty source file") 172 | return true 173 | } 174 | } 175 | return false 176 | } 177 | -------------------------------------------------------------------------------- /pkg/leeway/gitinfo_test.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/google/go-cmp/cmp" 8 | ) 9 | 10 | func TestParseGitStatus(t *testing.T) { 11 | type Expectation struct { 12 | Files map[string]struct{} 13 | Err string 14 | } 15 | tests := []struct { 16 | Name string 17 | In string 18 | Expectation Expectation 19 | }{ 20 | { 21 | Name: "empty input", 22 | Expectation: Expectation{}, 23 | }, 24 | { 25 | Name: "garbage", 26 | In: "hello world, this is garbage\nand some more", 27 | Expectation: Expectation{ 28 | Err: `cannot parse git status "hello world, this is garbage": expected two segments, got 5`, 29 | }, 30 | }, 31 | { 32 | Name: "valid input", 33 | In: " M foobar\n M this/is/a/file", 34 | Expectation: Expectation{ 35 | Files: map[string]struct{}{ 36 | "foobar": {}, 37 | "this/is/a/file": {}, 38 | }, 39 | }, 40 | }, 41 | } 42 | for _, test := range tests { 43 | t.Run(test.Name, func(t *testing.T) { 44 | files, err := parseGitStatus([]byte(test.In)) 45 | var act Expectation 46 | act.Files = files 47 | if err != nil { 48 | act.Err = err.Error() 49 | } 50 | if diff := cmp.Diff(test.Expectation, act); diff != "" { 51 | t.Errorf("ParseGitStatus() mismatch (-want +got):\n%s", diff) 52 | } 53 | }) 54 | } 55 | } 56 | 57 | func TestGitInfoDirtyFiles(t *testing.T) { 58 | tests := []struct { 59 | Name string 60 | In *GitInfo 61 | Files []string 62 | Expectation bool 63 | }{ 64 | { 65 | Name: "empty input", 66 | In: &GitInfo{}, 67 | Expectation: false, 68 | }, 69 | { 70 | Name: "dirty working copy", 71 | In: &GitInfo{ 72 | dirty: true, 73 | }, 74 | Files: []string{"foo"}, 75 | Expectation: true, 76 | }, 77 | { 78 | Name: "dirty file", 79 | In: &GitInfo{ 80 | dirty: true, 81 | dirtyFiles: map[string]struct{}{ 82 | "foo": {}, 83 | }, 84 | }, 85 | Files: []string{"foo"}, 86 | Expectation: true, 87 | }, 88 | { 89 | Name: "dirty file loc", 90 | In: &GitInfo{ 91 | WorkingCopyLoc: "bar/", 92 | dirty: true, 93 | dirtyFiles: map[string]struct{}{ 94 | "foo": {}, 95 | }, 96 | }, 97 | Files: []string{"bar/foo"}, 98 | Expectation: true, 99 | }, 100 | { 101 | Name: "unknown file", 102 | In: &GitInfo{ 103 | WorkingCopyLoc: "bar/", 104 | dirty: true, 105 | dirtyFiles: map[string]struct{}{ 106 | "foo": {}, 107 | }, 108 | }, 109 | Files: []string{"not/in/this/working/copy"}, 110 | Expectation: true, 111 | }, 112 | { 113 | Name: "clean file", 114 | In: &GitInfo{ 115 | dirty: true, 116 | dirtyFiles: map[string]struct{}{ 117 | "foo": {}, 118 | }, 119 | }, 120 | Files: []string{"bar"}, 121 | Expectation: false, 122 | }, 123 | } 124 | for _, test := range tests { 125 | t.Run(test.Name, func(t *testing.T) { 126 | act := test.In.DirtyFiles(test.Files) 127 | if diff := cmp.Diff(test.Expectation, act); diff != "" { 128 | t.Errorf("ParseGitStatus() mismatch (-want +got):\n%s", diff) 129 | } 130 | }) 131 | } 132 | } 133 | 134 | func TestGitError(t *testing.T) { 135 | err := &GitError{ 136 | Op: "status", 137 | Err: fmt.Errorf("command failed"), 138 | } 139 | expected := "git operation status failed: command failed" 140 | if err.Error() != expected { 141 | t.Errorf("GitError.Error() = %q, want %q", err.Error(), expected) 142 | } 143 | } 144 | 145 | func TestGitInfoIsDirty(t *testing.T) { 146 | tests := []struct { 147 | name string 148 | info *GitInfo 149 | want bool 150 | }{ 151 | { 152 | name: "clean working copy", 153 | info: &GitInfo{dirty: false}, 154 | want: false, 155 | }, 156 | { 157 | name: "dirty working copy", 158 | info: &GitInfo{dirty: true}, 159 | want: true, 160 | }, 161 | } 162 | 163 | for _, tt := range tests { 164 | t.Run(tt.name, func(t *testing.T) { 165 | if got := tt.info.IsDirty(); got != tt.want { 166 | t.Errorf("GitInfo.IsDirty() = %v, want %v", got, tt.want) 167 | } 168 | }) 169 | } 170 | } 171 | 172 | func TestGitInfoHasDirtyFile(t *testing.T) { 173 | tests := []struct { 174 | name string 175 | info *GitInfo 176 | file string 177 | want bool 178 | }{ 179 | { 180 | name: "clean working copy", 181 | info: &GitInfo{dirty: false}, 182 | file: "foo", 183 | want: false, 184 | }, 185 | { 186 | name: "dirty working copy no files tracked", 187 | info: &GitInfo{ 188 | dirty: true, 189 | dirtyFiles: nil, 190 | }, 191 | file: "foo", 192 | want: true, 193 | }, 194 | { 195 | name: "dirty working copy with specific file", 196 | info: &GitInfo{ 197 | dirty: true, 198 | dirtyFiles: map[string]struct{}{ 199 | "foo": {}, 200 | }, 201 | }, 202 | file: "foo", 203 | want: true, 204 | }, 205 | { 206 | name: "dirty working copy with clean file", 207 | info: &GitInfo{ 208 | dirty: true, 209 | dirtyFiles: map[string]struct{}{ 210 | "foo": {}, 211 | }, 212 | }, 213 | file: "bar", 214 | want: false, 215 | }, 216 | { 217 | name: "dirty working copy with file in working copy", 218 | info: &GitInfo{ 219 | WorkingCopyLoc: "/path/to", 220 | dirty: true, 221 | dirtyFiles: map[string]struct{}{ 222 | "foo": {}, 223 | }, 224 | }, 225 | file: "/path/to/foo", 226 | want: true, 227 | }, 228 | } 229 | 230 | for _, tt := range tests { 231 | t.Run(tt.name, func(t *testing.T) { 232 | if got := tt.info.HasDirtyFile(tt.file); got != tt.want { 233 | t.Errorf("GitInfo.HasDirtyFile() = %v, want %v", got, tt.want) 234 | } 235 | }) 236 | } 237 | } 238 | -------------------------------------------------------------------------------- /pkg/leeway/reporter_test.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "testing" 7 | "time" 8 | 9 | "github.com/google/go-cmp/cmp" 10 | ) 11 | 12 | func TestConsoleReporter(t *testing.T) { 13 | t.Parallel() 14 | 15 | type Expectation struct { 16 | Output string 17 | } 18 | 19 | start := time.Now() 20 | pkg := &Package{ 21 | C: &Component{ 22 | Name: "test", 23 | }, 24 | PackageInternal: PackageInternal{ 25 | Name: "test", 26 | }, 27 | } 28 | 29 | tests := []struct { 30 | Name string 31 | Reporter Reporter 32 | Func func(t *testing.T, r *ConsoleReporter) 33 | Expect Expectation 34 | }{ 35 | { 36 | Name: "all phases", 37 | Func: func(t *testing.T, r *ConsoleReporter) { 38 | r.PackageBuildStarted(pkg, "/tmp/build") 39 | 40 | r.now = func() time.Time { 41 | return start.Add(5 * time.Second) 42 | } 43 | r.PackageBuildFinished(pkg, &PackageBuildReport{ 44 | Phases: []PackageBuildPhase{ 45 | PackageBuildPhasePrep, 46 | PackageBuildPhasePull, 47 | PackageBuildPhaseLint, 48 | PackageBuildPhaseTest, 49 | PackageBuildPhaseBuild}, 50 | phaseEnter: map[PackageBuildPhase]time.Time{ 51 | PackageBuildPhasePrep: start, 52 | PackageBuildPhasePull: start.Add(time.Second), 53 | PackageBuildPhaseBuild: start.Add(2 * time.Second), 54 | PackageBuildPhaseTest: start.Add(3 * time.Second), 55 | PackageBuildPhaseLint: start.Add(4 * time.Second), 56 | }, 57 | phaseDone: map[PackageBuildPhase]time.Time{ 58 | PackageBuildPhasePrep: start.Add(time.Second), 59 | PackageBuildPhasePull: start.Add(2 * time.Second), 60 | PackageBuildPhaseBuild: start.Add(3 * time.Second), 61 | PackageBuildPhaseTest: start.Add(4 * time.Second), 62 | PackageBuildPhaseLint: start.Add(5 * time.Second), 63 | }, 64 | }) 65 | }, 66 | Expect: Expectation{ 67 | Output: `[test:test] build started (version unknown, builddir /tmp/build) 68 | [test:test] package build succeeded (5.00s) [prep: 1.0s | pull: 1.0s | lint: 1.0s | test: 1.0s | build: 1.0s] 69 | `, 70 | }, 71 | }, 72 | { 73 | Name: "no phases", 74 | Func: func(t *testing.T, r *ConsoleReporter) { 75 | r.PackageBuildStarted(pkg, "/tmp/build") 76 | r.PackageBuildFinished(pkg, &PackageBuildReport{ 77 | Phases: []PackageBuildPhase{}, 78 | }) 79 | }, 80 | Expect: Expectation{ 81 | Output: `[test:test] build started (version unknown, builddir /tmp/build) 82 | [test:test] package build succeeded (0.00s) 83 | `, 84 | }, 85 | }, 86 | { 87 | Name: "failed build", 88 | Func: func(t *testing.T, r *ConsoleReporter) { 89 | r.PackageBuildStarted(pkg, "/tmp/build") 90 | r.PackageBuildFinished(pkg, &PackageBuildReport{ 91 | Error: errors.New("failed"), 92 | }) 93 | }, 94 | Expect: Expectation{ 95 | Output: `[test:test] build started (version unknown, builddir /tmp/build) 96 | [test:test] package build failed while preping 97 | [test:test] Reason: failed 98 | `, 99 | }, 100 | }, 101 | } 102 | 103 | for _, test := range tests { 104 | t.Run(test.Name, func(t *testing.T) { 105 | t.Parallel() 106 | var ( 107 | act Expectation 108 | buf bytes.Buffer 109 | ) 110 | 111 | reporter := NewConsoleReporter() 112 | reporter.out = &buf 113 | reporter.now = func() time.Time { 114 | return start 115 | } 116 | 117 | test.Func(t, reporter) 118 | act.Output = buf.String() 119 | 120 | if diff := cmp.Diff(test.Expect.Output, act.Output); diff != "" { 121 | t.Errorf("output mismatch (-want +got):\n%s", diff) 122 | } 123 | }) 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /pkg/leeway/scripts.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "strings" 9 | "syscall" 10 | 11 | log "github.com/sirupsen/logrus" 12 | "golang.org/x/xerrors" 13 | ) 14 | 15 | // WorkdirLayout describes the layout of the working dir a script gets executed in 16 | type WorkdirLayout string 17 | 18 | const ( 19 | // WorkdirOrigin means the script is executed in the original location of the component where it's defined, 20 | // in the original workspace. 21 | WorkdirOrigin WorkdirLayout = "origin" 22 | 23 | // WorkdirPackages replicates the structure leeway produces during a package build based on the script's dependencies. 24 | WorkdirPackages WorkdirLayout = "packages" 25 | ) 26 | 27 | // ScriptType defines the type a script is of 28 | type ScriptType string 29 | 30 | const ( 31 | // BashScript means the script is executed by bash. 32 | // The shebang is added automatically. 33 | BashScript ScriptType = "bash" 34 | ) 35 | 36 | // Script is an executable, uncacheable unit that does not result in build artefacts 37 | type Script struct { 38 | C *Component 39 | 40 | Name string `yaml:"name"` 41 | Description string `yaml:"description"` 42 | Dependencies []string `yaml:"deps"` 43 | Environment []string `yaml:"env"` 44 | WorkdirLayout WorkdirLayout `yaml:"workdir"` 45 | Type ScriptType `yaml:"type"` 46 | Script string `yaml:"script"` 47 | 48 | dependencies []*Package 49 | } 50 | 51 | // FullName returns the packages fully qualified name (component:package) 52 | func (p *Script) FullName() string { 53 | return fmt.Sprintf("%s:%s", p.C.Name, p.Name) 54 | } 55 | 56 | // link connects resolves the references to the dependencies 57 | func (p *Script) link(idx map[string]*Package) error { 58 | p.dependencies = make([]*Package, len(p.Dependencies)) 59 | for i, dep := range p.Dependencies { 60 | var ok bool 61 | p.dependencies[i], ok = idx[dep] 62 | if !ok { 63 | return PackageNotFoundErr{dep} 64 | } 65 | } 66 | return nil 67 | } 68 | 69 | // GetDependencies returns the linked package dependencies or nil if not linked yet 70 | func (p *Script) GetDependencies() []*Package { 71 | return p.dependencies 72 | } 73 | 74 | // FilesystemSafeName returns a string that is safe to use in a Unix filesystem as directory or filename 75 | func (p *Script) FilesystemSafeName() string { 76 | pkgdir := p.FullName() 77 | pkgdir = strings.Replace(pkgdir, "/", "-", -1) 78 | pkgdir = strings.Replace(pkgdir, ":", "--", -1) 79 | // components in the workspace root would otherwise start with - which breaks a lot of shell commands 80 | pkgdir = strings.TrimLeft(pkgdir, "-") 81 | return pkgdir 82 | } 83 | 84 | // Run executes the script 85 | func (p *Script) Run(opts ...BuildOption) error { 86 | options, err := applyBuildOpts(opts) 87 | if err != nil { 88 | return err 89 | } 90 | buildCtx, err := newBuildContext(options) 91 | if err != nil { 92 | return err 93 | } 94 | 95 | unresolvedArgs, err := findUnresolvedArgumentsInScript(p) 96 | if err != nil { 97 | return err 98 | } 99 | if len(unresolvedArgs) != 0 { 100 | var msg string 101 | for _, arg := range unresolvedArgs { 102 | cleanArg := strings.TrimSuffix(strings.TrimPrefix(arg, "${"), "}") 103 | msg += fmt.Sprintf("cannot run script with unresolved argument \"%s\": use -D%s=value to set the argument\n\n", arg, cleanArg) 104 | } 105 | return xerrors.Errorf(msg) 106 | } 107 | 108 | if len(p.dependencies) > 0 { 109 | err = Build(&Package{ 110 | C: p.C, 111 | dependencies: p.dependencies, 112 | PackageInternal: PackageInternal{ 113 | Name: fmt.Sprintf("%s-deps", p.Name), 114 | Environment: p.Environment, 115 | Ephemeral: true, 116 | Type: GenericPackage, 117 | }, 118 | Config: GenericPkgConfig{}, 119 | }, withBuildContext(buildCtx)) 120 | if err != nil { 121 | return err 122 | } 123 | } 124 | 125 | tdir, deplocs, err := p.synthesizePackagesWorkdir(buildCtx) 126 | if err != nil { 127 | return err 128 | } 129 | 130 | paths := make([]string, 0, len(deplocs)) 131 | for _, pth := range deplocs { 132 | paths = append(paths, pth) 133 | } 134 | 135 | var wd string 136 | switch p.WorkdirLayout { 137 | case WorkdirPackages: 138 | wd = tdir 139 | case WorkdirOrigin: 140 | fallthrough 141 | default: 142 | wd = p.C.Origin 143 | } 144 | 145 | var ( 146 | env = append(os.Environ(), p.Environment...) 147 | pa bool 148 | ) 149 | for i, e := range env { 150 | if !strings.HasPrefix(e, "PATH=") { 151 | continue 152 | } 153 | pa = true 154 | 155 | if strings.TrimPrefix(e, "PATH=") != "" { 156 | e += ":" 157 | } 158 | e += strings.Join(paths, ":") 159 | env[i] = e 160 | } 161 | if !pa { 162 | env = append(env, fmt.Sprintf("PATH=%s", strings.Join(paths, ":"))) 163 | } 164 | for n, pth := range deplocs { 165 | env = append(env, fmt.Sprintf("%s=%s", strings.ToUpper(strings.ReplaceAll(n, "-", "_")), pth)) 166 | } 167 | 168 | // execute script 169 | switch p.Type { 170 | case BashScript: 171 | return executeBashScript(p.Script, wd, env) 172 | } 173 | 174 | return xerrors.Errorf("unknown script type: %s", p.Type) 175 | } 176 | 177 | // FindUnresolvedArguments finds any still unresolved build arguments in a set of packages 178 | func findUnresolvedArgumentsInScript(script *Script) ([]string, error) { 179 | args := buildArgRegexp.FindAll([]byte(script.Script), -1) 180 | vars := make(map[string]struct{}, len(args)) 181 | for _, arg := range args { 182 | vars[string(arg)] = struct{}{} 183 | } 184 | 185 | var res []string 186 | for v := range vars { 187 | res = append(res, v) 188 | } 189 | return res, nil 190 | } 191 | 192 | func (p *Script) synthesizePackagesWorkdir(buildCtx *buildContext) (path string, bins map[string]string, err error) { 193 | path, err = os.MkdirTemp(buildCtx.buildDir, fmt.Sprintf("script-%s-*", p.FilesystemSafeName())) 194 | if err != nil { 195 | return 196 | } 197 | 198 | bins = make(map[string]string, len(p.dependencies)) 199 | for _, dep := range p.dependencies { 200 | br, exists := buildCtx.LocalCache.Location(dep) 201 | if !exists { 202 | err = xerrors.Errorf("dependency %s is not built", dep.FullName()) 203 | return 204 | } 205 | 206 | loc := filepath.Join(path, dep.FilesystemSafeName()) 207 | err = os.MkdirAll(loc, 0755) 208 | if err != nil { 209 | return 210 | } 211 | 212 | var untarCmd []string 213 | untarCmd, err = BuildUnTarCommand( 214 | WithInputFile(br), 215 | WithTargetDir(loc), 216 | WithAutoDetectCompression(true), 217 | ) 218 | if err != nil { 219 | err = xerrors.Errorf("cannot build untar command for %s: %w", dep.FullName(), err) 220 | return 221 | } 222 | 223 | var out []byte 224 | cmd := exec.Command(untarCmd[0], untarCmd[1:]...) 225 | cmd.Dir = loc 226 | out, err = cmd.CombinedOutput() 227 | if err != nil { 228 | err = xerrors.Errorf("cannot unarchive build result for %s: %s", dep.FullName(), string(out)) 229 | return 230 | } 231 | 232 | bins[dep.FilesystemSafeName()] = loc 233 | } 234 | 235 | return 236 | } 237 | 238 | func executeBashScript(script string, wd string, env []string) error { 239 | f, err := os.CreateTemp("", "*.sh") 240 | if err != nil { 241 | return err 242 | } 243 | defer os.Remove(f.Name()) 244 | 245 | _, err = f.WriteString("#!/bin/bash\n") 246 | if err != nil { 247 | return err 248 | } 249 | _, err = f.WriteString(script) 250 | if err != nil { 251 | return err 252 | } 253 | f.Close() 254 | 255 | log.WithField("env", env).WithField("wd", wd).Debug("running bash script") 256 | 257 | cmd := exec.Command("bash", f.Name()) 258 | cmd.Env = env 259 | cmd.Dir = wd 260 | cmd.Stdin = os.Stdin 261 | cmd.Stderr = os.Stderr 262 | cmd.Stdout = os.Stdout 263 | 264 | err = cmd.Run() 265 | if exiterr, ok := err.(*exec.ExitError); ok { 266 | if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { 267 | return xerrors.Errorf("failed with exit code %d", status.ExitStatus()) 268 | } 269 | } 270 | return err 271 | } 272 | -------------------------------------------------------------------------------- /pkg/leeway/scripts_test.go: -------------------------------------------------------------------------------- 1 | package leeway_test 2 | 3 | import ( 4 | // "path/filepath" 5 | 6 | "fmt" 7 | "testing" 8 | 9 | "github.com/gitpod-io/leeway/pkg/leeway" 10 | "github.com/gitpod-io/leeway/pkg/testutil" 11 | ) 12 | 13 | // Used in multiple tests to verify how we deal with scripts having dependencies 14 | var genericPackage = leeway.Package{ 15 | PackageInternal: leeway.PackageInternal{ 16 | Name: "something", 17 | Type: "generic", 18 | Sources: []string{"*.txt"}, 19 | }, 20 | Config: leeway.GenericPkgConfig{ 21 | Commands: [][]string{{"echo"}}, 22 | Test: [][]string{{"echo", "testing"}}, 23 | }, 24 | } 25 | 26 | func TestScriptArgs(t *testing.T) { 27 | testutil.RunDUT() 28 | 29 | setup := &testutil.Setup{ 30 | Components: []testutil.Component{ 31 | { 32 | Location: "scripts", 33 | Packages: []leeway.Package{}, 34 | Scripts: []leeway.Script{ 35 | { 36 | Name: "echo", 37 | Description: "echos an argument", 38 | Script: `echo ${msg}`, 39 | }, 40 | }, 41 | }, 42 | }, 43 | } 44 | 45 | tests := []*testutil.CommandFixtureTest{ 46 | // If the argument isn't passed then Leeway should fail with an exit code of 1 47 | { 48 | Name: "unresolved arg", 49 | T: t, 50 | Args: []string{"run", "scripts:echo"}, 51 | NoNestedWorkspace: true, 52 | ExitCode: 1, 53 | Fixture: setup, 54 | }, 55 | // The argument should be passed to the script correctly - verified by checking stdout 56 | { 57 | Name: "resovled args", 58 | T: t, 59 | Args: []string{"run", "scripts:echo", "-Dmsg=foobar"}, 60 | NoNestedWorkspace: true, 61 | ExitCode: 0, 62 | StdoutSubs: []string{"foobar"}, 63 | Fixture: setup, 64 | }, 65 | } 66 | 67 | for _, test := range tests { 68 | test.Run() 69 | } 70 | } 71 | 72 | func TestWorkingDirLayout(t *testing.T) { 73 | testutil.RunDUT() 74 | 75 | setup := &testutil.Setup{ 76 | Components: []testutil.Component{ 77 | { 78 | Location: "scripts", 79 | Packages: []leeway.Package{genericPackage}, 80 | Scripts: []leeway.Script{ 81 | { 82 | Name: "pwd-origin", 83 | WorkdirLayout: "origin", 84 | Dependencies: []string{fmt.Sprintf(":%s", genericPackage.Name)}, 85 | Script: `pwd && find .`, 86 | }, 87 | { 88 | Name: "pwd-packages", 89 | WorkdirLayout: "packages", 90 | Dependencies: []string{fmt.Sprintf(":%s", genericPackage.Name)}, 91 | Script: `pwd && find .`, 92 | }, 93 | }, 94 | }, 95 | }, 96 | } 97 | 98 | tests := []*testutil.CommandFixtureTest{ 99 | // Shows that even though it depends on a package, it isn't copied into the working directory 100 | { 101 | Name: "origin", 102 | T: t, 103 | Args: []string{"run", "scripts:pwd-origin"}, 104 | ExitCode: 0, 105 | NoNestedWorkspace: true, 106 | Fixture: setup, 107 | StdoutSubs: []string{`. 108 | ./BUILD.yaml`}, 109 | }, 110 | // Shows that the dependency is copied into the working directly, and that nothing else is present 111 | { 112 | Name: "packages", 113 | T: t, 114 | Args: []string{"run", "scripts:pwd-packages"}, 115 | ExitCode: 0, 116 | NoNestedWorkspace: true, 117 | Fixture: setup, 118 | StdoutSubs: []string{`. 119 | ./scripts--something`}, 120 | }, 121 | } 122 | 123 | for _, test := range tests { 124 | test.Run() 125 | } 126 | } 127 | 128 | func TestPATHEnvironment(t *testing.T) { 129 | testutil.RunDUT() 130 | 131 | setup := &testutil.Setup{ 132 | Components: []testutil.Component{ 133 | { 134 | Location: "scripts", 135 | Packages: []leeway.Package{genericPackage}, 136 | Scripts: []leeway.Script{ 137 | { 138 | Name: "path", 139 | Description: "prints the $PATH of the script execution context", 140 | Dependencies: []string{fmt.Sprintf(":%s", genericPackage.Name)}, 141 | Script: `echo $PATH`, 142 | }, 143 | }, 144 | }, 145 | }, 146 | } 147 | 148 | tests := []*testutil.CommandFixtureTest{ 149 | // The PATH should contain the package that the script depends on 150 | { 151 | Name: "resovled args", 152 | T: t, 153 | Args: []string{"run", "scripts:path"}, 154 | NoNestedWorkspace: true, 155 | ExitCode: 0, 156 | StdoutSubs: []string{"scripts--something"}, 157 | Fixture: setup, 158 | }, 159 | } 160 | 161 | for _, test := range tests { 162 | test.Run() 163 | } 164 | } 165 | 166 | func TestScriptParallel(t *testing.T) { 167 | testutil.RunDUT() 168 | 169 | setup := &testutil.Setup{ 170 | Components: []testutil.Component{ 171 | { 172 | Location: "scripts", 173 | Packages: []leeway.Package{}, 174 | Scripts: []leeway.Script{ 175 | { 176 | Name: "script-a", 177 | Description: "Script A", 178 | Script: `echo "Starting script A"`, 179 | }, 180 | { 181 | Name: "script-b", 182 | Description: "Script B", 183 | Script: `echo "Starting script B"`, 184 | }, 185 | { 186 | Name: "exit-42", 187 | Description: "Exists with exit code 42", 188 | Script: `echo "Exiting" && exit 42`, 189 | }, 190 | }, 191 | }, 192 | }, 193 | } 194 | 195 | tests := []*testutil.CommandFixtureTest{ 196 | // When two or more scripts are passed, it should execute both. 197 | { 198 | Name: "two successful invocations", 199 | T: t, 200 | Args: []string{"run", "scripts:script-a", "scripts:script-b"}, 201 | NoNestedWorkspace: true, 202 | ExitCode: 0, 203 | Fixture: setup, 204 | StdoutSubs: []string{"Starting script A", "Starting script B"}, 205 | }, 206 | // When one of the scripts fail, Leeway should still run the other scripts to completions 207 | // and it should fail with an exit code of 1 208 | // 209 | { 210 | Name: "two successful invocations", 211 | T: t, 212 | Args: []string{"run", "scripts:exit-42", "scripts:script-a"}, 213 | NoNestedWorkspace: true, 214 | ExitCode: 1, 215 | Fixture: setup, 216 | StdoutSubs: []string{"Starting script A", "Exiting"}, 217 | }, 218 | } 219 | 220 | for _, test := range tests { 221 | test.Run() 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /pkg/leeway/version.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | // Version is the version of this leeway build 4 | var Version string = "unknown" 5 | -------------------------------------------------------------------------------- /pkg/leeway/watcher.go: -------------------------------------------------------------------------------- 1 | package leeway 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | "strings" 7 | "sync" 8 | "time" 9 | 10 | "github.com/fsnotify/fsnotify" 11 | log "github.com/sirupsen/logrus" 12 | 13 | "github.com/gitpod-io/leeway/pkg/doublestar" 14 | ) 15 | 16 | // WatchSources watches the source files of the packages until the context is done 17 | func WatchSources(ctx context.Context, pkgs []*Package, debounceDuration time.Duration) (changed <-chan struct{}, errs <-chan error) { 18 | var ( 19 | rawChng = make(chan struct{}) 20 | chng = make(chan struct{}) 21 | errchan = make(chan error, 1) 22 | ) 23 | changed = chng 24 | errs = errchan 25 | 26 | watcher, err := fsnotify.NewWatcher() 27 | if err != nil { 28 | errchan <- err 29 | return 30 | } 31 | 32 | var ( 33 | matcher []*pathMatcher 34 | folders = make(map[string]*Package) 35 | ) 36 | for _, pkg := range pkgs { 37 | for _, src := range pkg.Sources { 38 | folders[filepath.Dir(src)] = pkg 39 | } 40 | } 41 | for f, pkg := range folders { 42 | log.WithField("path", f).Debug("adding watcher") 43 | //nolint:errcheck 44 | watcher.Add(f) 45 | 46 | matcher = append(matcher, &pathMatcher{ 47 | Base: f, 48 | Patterns: pkg.originalSources, 49 | }) 50 | } 51 | 52 | if debounceDuration == 0 { 53 | go func() { 54 | for { 55 | select { 56 | case <-rawChng: 57 | chng <- struct{}{} 58 | case <-ctx.Done(): 59 | return 60 | } 61 | } 62 | }() 63 | } else { 64 | var ( 65 | c int64 66 | mu sync.RWMutex 67 | ) 68 | 69 | go func() { 70 | for { 71 | select { 72 | case <-ctx.Done(): 73 | return 74 | case <-rawChng: 75 | mu.Lock() 76 | c++ 77 | key := c 78 | mu.Unlock() 79 | 80 | go func() { 81 | <-time.After(debounceDuration) 82 | mu.RLock() 83 | defer mu.RUnlock() 84 | 85 | if c != key { 86 | return 87 | } 88 | chng <- struct{}{} 89 | }() 90 | } 91 | } 92 | }() 93 | } 94 | 95 | go func() { 96 | defer watcher.Close() 97 | for { 98 | select { 99 | case evt := <-watcher.Events: 100 | var ( 101 | patterns []string 102 | matches bool 103 | ) 104 | for _, m := range matcher { 105 | if m.Matches(evt.Name) { 106 | matches = true 107 | patterns = m.Patterns 108 | break 109 | } 110 | } 111 | if !matches { 112 | log.WithField("path", evt.Name).Debug("dismissed file event that did not match source globs") 113 | return 114 | } 115 | 116 | dfn := filepath.Dir(evt.Name) 117 | if _, ok := folders[dfn]; !ok { 118 | matcher = append(matcher, &pathMatcher{ 119 | Base: dfn, 120 | Patterns: patterns, 121 | }) 122 | //nolint:errcheck 123 | watcher.Add(dfn) 124 | log.WithField("path", dfn).Debug("added new source folder") 125 | } 126 | 127 | log.WithField("path", evt.Name).Debug("source file changed") 128 | rawChng <- struct{}{} 129 | case err := <-watcher.Errors: 130 | errchan <- err 131 | case <-ctx.Done(): 132 | return 133 | } 134 | } 135 | }() 136 | 137 | return 138 | } 139 | 140 | type pathMatcher struct { 141 | Base string 142 | Patterns []string 143 | } 144 | 145 | func (pm *pathMatcher) Matches(path string) (matches bool) { 146 | if !strings.HasPrefix(path, pm.Base) { 147 | return false 148 | } 149 | for _, p := range pm.Patterns { 150 | matches, _ := doublestar.Match(p, strings.TrimPrefix(strings.TrimPrefix(path, pm.Base), "/")) 151 | if matches { 152 | return true 153 | } 154 | } 155 | return false 156 | } 157 | -------------------------------------------------------------------------------- /pkg/linker/yarn.go: -------------------------------------------------------------------------------- 1 | package linker 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "strings" 9 | 10 | log "github.com/sirupsen/logrus" 11 | "golang.org/x/xerrors" 12 | 13 | "github.com/gitpod-io/leeway/pkg/leeway" 14 | ) 15 | 16 | // LinkYarnPackagesWithYarn2 uses `yarn link` to link all TS packages in-situ. 17 | func LinkYarnPackagesWithYarn2(workspace *leeway.Workspace) error { 18 | var ( 19 | pkgIdx = make(map[string]string) 20 | pkgJSONIdx = make(map[string]string) 21 | ) 22 | for n, p := range workspace.Packages { 23 | if p.Type != leeway.YarnPackage { 24 | continue 25 | } 26 | 27 | var pkgjsonFn string 28 | for _, src := range p.Sources { 29 | if strings.HasSuffix(src, "/package.json") { 30 | pkgjsonFn = src 31 | break 32 | } 33 | } 34 | if pkgjsonFn == "" { 35 | log.WithField("pkg", n).Warn("no package.json found - skipping") 36 | continue 37 | } 38 | pkgJSONIdx[n] = pkgjsonFn 39 | 40 | fc, err := os.ReadFile(pkgjsonFn) 41 | if err != nil { 42 | return err 43 | } 44 | var pkgjson struct { 45 | Name string `json:"name"` 46 | } 47 | err = json.Unmarshal(fc, &pkgjson) 48 | if err != nil { 49 | return err 50 | } 51 | pkgIdx[n] = pkgjson.Name 52 | } 53 | 54 | for n, p := range workspace.Packages { 55 | if p.Type != leeway.YarnPackage { 56 | continue 57 | } 58 | pkgjsonFn := pkgJSONIdx[n] 59 | 60 | fc, err := os.ReadFile(pkgjsonFn) 61 | if err != nil { 62 | return err 63 | } 64 | var pkgjson map[string]interface{} 65 | err = json.Unmarshal(fc, &pkgjson) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | var resolutions map[string]interface{} 71 | if res, ok := pkgjson["resolutions"]; ok { 72 | resolutions, ok = res.(map[string]interface{}) 73 | if !ok { 74 | return xerrors.Errorf("%s: found resolutions but they're not a map", n) 75 | } 76 | } else { 77 | resolutions = make(map[string]interface{}) 78 | } 79 | for _, dep := range p.GetTransitiveDependencies() { 80 | if dep.Type != leeway.YarnPackage { 81 | continue 82 | } 83 | 84 | yarnPkg, ok := pkgIdx[dep.FullName()] 85 | if !ok { 86 | log.WithField("dep", dep.FullName()).WithField("pkg", n).Warn("did not find yarn package name - linking might be broken") 87 | continue 88 | } 89 | resolutions[yarnPkg] = fmt.Sprintf("portal://%s", dep.C.Origin) 90 | } 91 | if len(resolutions) > 0 { 92 | pkgjson["resolutions"] = resolutions 93 | } 94 | 95 | fd, err := os.OpenFile(pkgjsonFn, os.O_TRUNC|os.O_WRONLY, 0644) 96 | if err != nil { 97 | return err 98 | } 99 | enc := json.NewEncoder(fd) 100 | enc.SetEscapeHTML(false) 101 | enc.SetIndent("", " ") 102 | err = enc.Encode(pkgjson) 103 | fd.Close() 104 | if err != nil { 105 | return err 106 | } 107 | 108 | log.WithField("pkg", n).WithField("resolutions", resolutions).Debug("linked package") 109 | } 110 | 111 | var lerr error 112 | for n, p := range workspace.Packages { 113 | if p.Type != leeway.YarnPackage { 114 | continue 115 | } 116 | 117 | cmd := exec.Command("yarn") 118 | log.WithField("pkg", n).WithField("cwd", p.C.Origin).WithField("cmd", "yarn").Debug("running yarn") 119 | cmd.Dir = p.C.Origin 120 | cmd.Stdout = os.Stdout 121 | cmd.Stdin = os.Stdin 122 | cmd.Stderr = os.Stderr 123 | err := cmd.Run() 124 | if err != nil { 125 | log.WithError(err).Error("error while running yarn") 126 | lerr = xerrors.Errorf("yarn failed for %s: %w", n, err) 127 | } 128 | } 129 | 130 | return lerr 131 | } 132 | -------------------------------------------------------------------------------- /pkg/prettyprint/prettyprint.go: -------------------------------------------------------------------------------- 1 | package prettyprint 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "text/tabwriter" 8 | "text/template" 9 | 10 | "gopkg.in/yaml.v3" 11 | ) 12 | 13 | // Format is an output format for pretty printing 14 | type Format string 15 | 16 | const ( 17 | // TemplateFormat produces text/template-based output 18 | TemplateFormat Format = "template" 19 | // JSONFormat produces JSON output 20 | JSONFormat Format = "json" 21 | // YAMLFormat produces YAML output 22 | YAMLFormat Format = "yaml" 23 | ) 24 | 25 | // Writer preconfigures the write function 26 | type Writer struct { 27 | Out io.Writer 28 | Format Format 29 | FormatString string 30 | } 31 | 32 | // Write prints the input in the preconfigred way 33 | func (w *Writer) Write(in interface{}) error { 34 | switch w.Format { 35 | case TemplateFormat: 36 | return writeTemplate(w.Out, in, w.FormatString) 37 | case JSONFormat: 38 | return json.NewEncoder(w.Out).Encode(in) 39 | case YAMLFormat: 40 | return yaml.NewEncoder(w.Out).Encode(in) 41 | default: 42 | return fmt.Errorf("unknown format: %s", w.Format) 43 | } 44 | } 45 | 46 | func writeTemplate(out io.Writer, in interface{}, tplc string) error { 47 | tpl := template.New("template") 48 | tpl, err := tpl.Parse(tplc) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) 54 | defer w.Flush() 55 | 56 | return tpl.Execute(w, in) 57 | } 58 | -------------------------------------------------------------------------------- /pkg/provutil/access.go: -------------------------------------------------------------------------------- 1 | package provutil 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | 7 | "sigs.k8s.io/bom/pkg/provenance" 8 | ) 9 | 10 | // DecodeBundle returns a function which attempts to decode an attestation bundle from the reader 11 | // and calls the handler for every envelope found in the bundle. If decoding fails, or the handler 12 | // returns an error, decoding stops and the error is returned. 13 | func DecodeBundle(bundle io.Reader, handler func(env *provenance.Envelope) error) error { 14 | var env provenance.Envelope 15 | dec := json.NewDecoder(bundle) 16 | for dec.More() { 17 | err := dec.Decode(&env) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | err = handler(&env) 23 | if err != nil { 24 | return err 25 | } 26 | } 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /pkg/provutil/assert.go: -------------------------------------------------------------------------------- 1 | package provutil 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/gitpod-io/leeway/pkg/leeway" 9 | "github.com/in-toto/in-toto-golang/in_toto" 10 | log "github.com/sirupsen/logrus" 11 | "sigs.k8s.io/bom/pkg/provenance" 12 | ) 13 | 14 | type Assertion struct { 15 | Name string 16 | Description string 17 | Run func(stmt *provenance.Statement) []Violation 18 | RunBundle func(bundle *provenance.Envelope) []Violation 19 | } 20 | 21 | type Violation struct { 22 | Assertion *Assertion 23 | Statement *provenance.Statement 24 | Desc string 25 | } 26 | 27 | func (v Violation) String() string { 28 | if v.Statement == nil { 29 | return fmt.Sprintf("failed %s: %s", v.Assertion.Name, v.Desc) 30 | } 31 | 32 | pred := v.Statement.Predicate 33 | return fmt.Sprintf("%s failed %s: %s", pred.Invocation.ConfigSource.EntryPoint, v.Assertion.Name, v.Desc) 34 | } 35 | 36 | type Assertions []*Assertion 37 | 38 | func (a Assertions) AssertBundle(bundle *provenance.Envelope) (failed []Violation) { 39 | for _, as := range a { 40 | if as.RunBundle == nil { 41 | continue 42 | } 43 | 44 | res := as.RunBundle(bundle) 45 | for i := range res { 46 | res[i].Assertion = as 47 | } 48 | failed = append(failed, res...) 49 | } 50 | return 51 | } 52 | 53 | func (a Assertions) AssertStatement(stmt *provenance.Statement) (failed []Violation) { 54 | // we must not keep a reference to stmt around - it will change for each invocation 55 | s := *stmt 56 | for _, as := range a { 57 | if as.Run == nil { 58 | continue 59 | } 60 | 61 | res := as.Run(stmt) 62 | for i := range res { 63 | res[i].Statement = &s 64 | res[i].Assertion = as 65 | } 66 | failed = append(failed, res...) 67 | } 68 | return 69 | } 70 | 71 | var AssertBuiltWithLeeway = &Assertion{ 72 | Name: "built-with-leeway", 73 | Description: "ensures all bundle entries have been built with leeway", 74 | Run: func(stmt *provenance.Statement) []Violation { 75 | pred := stmt.Predicate 76 | if strings.HasPrefix(pred.Builder.ID, leeway.ProvenanceBuilderID) { 77 | return nil 78 | } 79 | 80 | return []Violation{ 81 | {Desc: "was not built using leeway"}, 82 | } 83 | }, 84 | } 85 | 86 | func AssertBuiltWithLeewayVersion(version string) *Assertion { 87 | return &Assertion{ 88 | Name: "built-with-leeway-version", 89 | Description: "ensures all bundle entries which have been built using leeway, used version " + version, 90 | Run: func(stmt *provenance.Statement) []Violation { 91 | pred := stmt.Predicate 92 | if !strings.HasPrefix(pred.Builder.ID, leeway.ProvenanceBuilderID) { 93 | return nil 94 | } 95 | 96 | if pred.Builder.ID != leeway.ProvenanceBuilderID+":"+version { 97 | return []Violation{{Desc: "was built using leeway version " + strings.TrimPrefix(pred.Builder.ID, leeway.ProvenanceBuilderID+":")}} 98 | } 99 | 100 | return nil 101 | }, 102 | } 103 | } 104 | 105 | var AssertGitMaterialOnly = &Assertion{ 106 | Name: "git-material-only", 107 | Description: "ensures all subjects were built from Git material only", 108 | Run: func(stmt *provenance.Statement) []Violation { 109 | pred := stmt.Predicate 110 | for _, m := range pred.Materials { 111 | if strings.HasPrefix(m.URI, "git+") || strings.HasPrefix(m.URI, "git://") { 112 | continue 113 | } 114 | 115 | return []Violation{{ 116 | Desc: "contains non-Git material, e.g. " + m.URI, 117 | }} 118 | } 119 | return nil 120 | }, 121 | } 122 | 123 | func AssertSignedWith(key in_toto.Key) *Assertion { 124 | return &Assertion{ 125 | Name: "signed-with", 126 | Description: "ensures all bundles are signed with the given key", 127 | RunBundle: func(bundle *provenance.Envelope) []Violation { 128 | for _, s := range bundle.Signatures { 129 | raw, err := json.Marshal(s) 130 | if err != nil { 131 | return []Violation{{Desc: "assertion error: " + err.Error()}} 132 | } 133 | var sig in_toto.Signature 134 | err = json.Unmarshal(raw, &sig) 135 | if err != nil { 136 | return []Violation{{Desc: "assertion error: " + err.Error()}} 137 | } 138 | 139 | err = in_toto.VerifySignature(key, sig, []byte(bundle.Payload)) 140 | if err != nil { 141 | log.WithError(err).WithField("signature", sig).Debug("signature does not match") 142 | continue 143 | } 144 | 145 | return nil 146 | } 147 | return []Violation{{Desc: "not signed with the given key"}} 148 | }, 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /pkg/testutil/testutil.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "flag" 7 | "io" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "strings" 12 | "testing" 13 | 14 | "github.com/gitpod-io/leeway/cmd" 15 | "github.com/gitpod-io/leeway/pkg/leeway" 16 | "gopkg.in/yaml.v3" 17 | ) 18 | 19 | type Setup struct { 20 | Workspace leeway.Workspace `yaml:"workspace"` 21 | Components []Component `yaml:"components"` 22 | Files map[string]string `yaml:"files"` 23 | } 24 | 25 | type Component struct { 26 | Location string `yaml:"location"` 27 | Files map[string]string `yaml:"files"` 28 | Comp leeway.Component `yaml:"comp"` 29 | Packages []leeway.Package `yaml:"packages"` 30 | Scripts []leeway.Script `yaml:"scripts"` 31 | } 32 | 33 | // LoadFromYAML loads a workspace setup from a YAML file 34 | func LoadFromYAML(in io.Reader) (*Setup, error) { 35 | fc, err := io.ReadAll(in) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | var res Setup 41 | err = yaml.Unmarshal(fc, &res) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | return &res, nil 47 | } 48 | 49 | // Materialize produces a leeway workspace according to the setup spec 50 | func (s Setup) Materialize() (workspaceRoot string, err error) { 51 | workspaceRoot, err = os.MkdirTemp("", "leeway-test-*") 52 | if err != nil { 53 | return 54 | } 55 | 56 | fc, err := yaml.Marshal(s.Workspace) 57 | if err != nil { 58 | return 59 | } 60 | err = os.WriteFile(filepath.Join(workspaceRoot, "WORKSPACE.yaml"), fc, 0644) 61 | if err != nil { 62 | return 63 | } 64 | for fn, content := range s.Files { 65 | fn = filepath.Join(workspaceRoot, fn) 66 | err = os.MkdirAll(filepath.Dir(fn), 0755) 67 | if errors.Is(err, os.ErrExist) { 68 | err = nil 69 | } 70 | if err != nil { 71 | return 72 | } 73 | err = os.WriteFile(fn, []byte(content), 0644) 74 | if err != nil { 75 | return 76 | } 77 | } 78 | 79 | for _, comp := range s.Components { 80 | err = os.MkdirAll(filepath.Join(workspaceRoot, comp.Location), 0755) 81 | if err != nil { 82 | return 83 | } 84 | 85 | cmp := struct { 86 | Constants leeway.Arguments `yaml:"const,omitempty"` 87 | Packages []leeway.Package `yaml:"packages,omitempty"` 88 | Scripts []leeway.Script `yaml:"scripts,omitempty"` 89 | }{ 90 | Constants: comp.Comp.Constants, 91 | Packages: comp.Packages, 92 | Scripts: comp.Scripts, 93 | } 94 | 95 | fc, err = yaml.Marshal(cmp) 96 | if err != nil { 97 | return 98 | } 99 | 100 | err = os.WriteFile(filepath.Join(workspaceRoot, comp.Location, "BUILD.yaml"), fc, 0644) 101 | if err != nil { 102 | return 103 | } 104 | 105 | for fn, content := range comp.Files { 106 | err = os.MkdirAll(filepath.Join(workspaceRoot, comp.Location, filepath.Dir(fn)), 0755) 107 | if errors.Is(err, os.ErrExist) { 108 | err = nil 109 | } 110 | if err != nil { 111 | return 112 | } 113 | err = os.WriteFile(filepath.Join(workspaceRoot, comp.Location, fn), []byte(content), 0644) 114 | if err != nil { 115 | return 116 | } 117 | } 118 | } 119 | 120 | return 121 | } 122 | 123 | var Dut = flag.Bool("dut", false, "run command/device under test") 124 | 125 | func RunDUT() { 126 | if *Dut { 127 | cmd.Execute() 128 | os.Exit(0) 129 | } 130 | } 131 | 132 | type CommandFixtureTest struct { 133 | Name string 134 | T *testing.T 135 | Args []string 136 | ExitCode int 137 | NoNestedWorkspace bool 138 | StdoutSubs []string 139 | NoStdoutSub string 140 | StderrSub string 141 | NoStderrSub string 142 | Eval func(t *testing.T, stdout, stderr string) 143 | Fixture *Setup 144 | FixturePath string 145 | } 146 | 147 | // Run executes the fixture test - do not forget to call this one 148 | func (ft *CommandFixtureTest) Run() { 149 | if *Dut { 150 | cmd.Execute() 151 | return 152 | } 153 | 154 | ft.T.Run(ft.Name, func(t *testing.T) { 155 | loc := "../../" 156 | 157 | if ft.FixturePath != "" && ft.Fixture != nil { 158 | t.Fatalf("Only one of FixturePath and Fixture must be set. You have set both.") 159 | } 160 | 161 | if ft.FixturePath != "" { 162 | fp, err := os.Open(ft.FixturePath) 163 | if err != nil { 164 | t.Fatalf("cannot load fixture from %s: %v", ft.FixturePath, err) 165 | } 166 | ft.Fixture, err = LoadFromYAML(fp) 167 | fp.Close() 168 | if err != nil { 169 | t.Fatalf("cannot load fixture from %s: %v", ft.FixturePath, err) 170 | } 171 | } 172 | if ft.Fixture != nil { 173 | var err error 174 | loc, err = ft.Fixture.Materialize() 175 | if err != nil { 176 | t.Fatalf("cannot materialize fixture: %v", err) 177 | } 178 | t.Logf("materialized fixture workspace: %s", loc) 179 | t.Cleanup(func() { os.RemoveAll(loc) }) 180 | } 181 | 182 | env := os.Environ() 183 | n := 0 184 | for _, x := range env { 185 | if strings.HasPrefix(x, "LEEWAY_") { 186 | continue 187 | } 188 | env[n] = x 189 | n++ 190 | } 191 | env = env[:n] 192 | 193 | self, err := os.Executable() 194 | if err != nil { 195 | t.Fatalf("cannot identify test binary: %q", err) 196 | } 197 | cmd := exec.Command(self, append([]string{"--dut"}, ft.Args...)...) 198 | var ( 199 | sout = bytes.NewBuffer(nil) 200 | serr = bytes.NewBuffer(nil) 201 | ) 202 | cmd.Stdout = sout 203 | cmd.Stderr = serr 204 | cmd.Dir = loc 205 | cmd.Env = env 206 | err = cmd.Run() 207 | 208 | var exitCode int 209 | if xerr, ok := err.(*exec.ExitError); ok { 210 | exitCode = xerr.ExitCode() 211 | err = nil 212 | } 213 | if err != nil { 214 | t.Fatalf("cannot re-run test binary: %q", err) 215 | } 216 | if exitCode != ft.ExitCode { 217 | t.Errorf("unepxected exit code: expected %d, actual %d (stderr: %s, stdout: %s)", ft.ExitCode, exitCode, serr.String(), sout.String()) 218 | } 219 | var ( 220 | stdout = sout.String() 221 | stderr = serr.String() 222 | ) 223 | for _, stdoutStub := range ft.StdoutSubs { 224 | if !strings.Contains(stdout, stdoutStub) { 225 | t.Errorf("stdout: expected to find \"%s\" in \"%s\"", stdoutStub, stdout) 226 | } 227 | } 228 | if ft.NoStdoutSub != "" && strings.Contains(stdout, ft.NoStdoutSub) { 229 | t.Errorf("stdout: expected not to find \"%s\" in \"%s\"", ft.NoStdoutSub, stdout) 230 | } 231 | if !strings.Contains(stderr, ft.StderrSub) { 232 | t.Errorf("stderr: expected to find \"%s\" in \"%s\"", ft.StderrSub, stderr) 233 | } 234 | if ft.NoStderrSub != "" && strings.Contains(stderr, ft.NoStderrSub) { 235 | t.Errorf("stderr: expected not to find \"%s\" in \"%s\"", ft.NoStderrSub, stderr) 236 | } 237 | if ft.Eval != nil { 238 | ft.Eval(t, stdout, stderr) 239 | } 240 | }) 241 | } 242 | -------------------------------------------------------------------------------- /pkg/testutil/testutil_test.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "testing" 11 | 12 | "github.com/gitpod-io/leeway/pkg/leeway" 13 | "github.com/google/go-cmp/cmp" 14 | "github.com/google/go-cmp/cmp/cmpopts" 15 | ) 16 | 17 | func TestLoadFromYAML(t *testing.T) { 18 | ignores := cmpopts.IgnoreUnexported( 19 | leeway.Workspace{}, 20 | leeway.Component{}, 21 | leeway.Package{}, 22 | leeway.WorkspaceProvenance{}, 23 | leeway.GitInfo{}, 24 | ) 25 | 26 | tests := []struct { 27 | Name string 28 | Content string 29 | Expectation *Setup 30 | }{ 31 | { 32 | Name: "empty", 33 | Expectation: &Setup{}, 34 | }, 35 | { 36 | Name: "single", 37 | Expectation: &Setup{ 38 | Components: []Component{ 39 | { 40 | Location: "comp1", 41 | Packages: []leeway.Package{ 42 | { 43 | PackageInternal: leeway.PackageInternal{ 44 | Name: "pkg1", 45 | Type: leeway.GenericPackage, 46 | Sources: []string{"**"}, 47 | }, 48 | Config: leeway.GenericPkgConfig{ 49 | Commands: [][]string{{"foo"}}, 50 | }, 51 | }, 52 | }, 53 | }, 54 | }, 55 | }, 56 | Content: `components: 57 | - location: comp1 58 | packages: 59 | - name: pkg1 60 | type: generic 61 | srcs: 62 | - "**" 63 | config: 64 | commands: [["foo"]]`, 65 | }, 66 | } 67 | 68 | for _, test := range tests { 69 | t.Run(test.Name, func(t *testing.T) { 70 | act, err := LoadFromYAML(bytes.NewBufferString(test.Content)) 71 | if err != nil { 72 | t.Fatal(err) 73 | } 74 | 75 | if diff := cmp.Diff(test.Expectation, act, ignores); diff != "" { 76 | t.Errorf("LoadFromYAML() mismatch (-want +got):\n%s", diff) 77 | } 78 | }) 79 | } 80 | } 81 | 82 | func TestMaterialise(t *testing.T) { 83 | type File struct { 84 | Path string 85 | Size int64 86 | SHA256 string 87 | } 88 | tests := []struct { 89 | Name string 90 | Setup Setup 91 | Expectation []File 92 | }{ 93 | { 94 | Name: "simple", 95 | Setup: Setup{ 96 | Components: []Component{ 97 | { 98 | Location: "comp1", 99 | Files: map[string]string{ 100 | "someFile": "content", 101 | "some/other/file": "more content", 102 | "some/other/another": "more content", 103 | }, 104 | Comp: leeway.Component{ 105 | Constants: leeway.Arguments{ 106 | "hello": "world", 107 | }, 108 | }, 109 | Packages: []leeway.Package{ 110 | { 111 | PackageInternal: leeway.PackageInternal{ 112 | Name: "pkg1", 113 | Type: leeway.GenericPackage, 114 | Sources: []string{"**"}, 115 | }, 116 | Config: leeway.GenericPkgConfig{ 117 | Commands: [][]string{{"bla"}}, 118 | }, 119 | }, 120 | }, 121 | }, 122 | }, 123 | }, 124 | Expectation: []File{ 125 | {Path: "WORKSPACE.yaml"}, 126 | {Path: "comp1/BUILD.yaml", SHA256: "6cc73a81aa8f00851c4738947d0d22ae296d8c626af4d1f4fe60fbabb09f906f"}, 127 | }, 128 | }, 129 | } 130 | 131 | for _, test := range tests { 132 | t.Run(test.Name, func(t *testing.T) { 133 | loc, err := test.Setup.Materialize() 134 | if err != nil { 135 | t.Fatal(err) 136 | } 137 | t.Cleanup(func() { os.RemoveAll(loc) }) 138 | t.Logf("materialized at %s", loc) 139 | 140 | for _, f := range test.Expectation { 141 | fn := filepath.Join(loc, f.Path) 142 | stat, err := os.Stat(fn) 143 | if err != nil { 144 | t.Errorf("expected file mismatch: %s: %v", f.Path, err) 145 | continue 146 | } 147 | 148 | if f.Size > 0 && stat.Size() != f.Size { 149 | t.Errorf("expected file size mismatch: %s: expected %d, got %d", f.Path, f.Size, stat.Size()) 150 | } 151 | 152 | if f.SHA256 == "" { 153 | continue 154 | } 155 | hash := sha256.New() 156 | fp, err := os.Open(fn) 157 | if err != nil { 158 | t.Errorf("cannot hash %s: %v", fn, err) 159 | continue 160 | } 161 | _, err = io.Copy(hash, fp) 162 | fp.Close() 163 | if err != nil { 164 | t.Errorf("cannot hash %s: %v", fn, err) 165 | continue 166 | } 167 | 168 | sum := fmt.Sprintf("%x", hash.Sum(nil)) 169 | if f.SHA256 != sum { 170 | t.Errorf("file hash mismatch: %s: expected %s, got %s", f.Path, f.SHA256, sum) 171 | } 172 | } 173 | }) 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /pkg/vet/components.go: -------------------------------------------------------------------------------- 1 | package vet 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/gitpod-io/leeway/pkg/leeway" 9 | ) 10 | 11 | func init() { 12 | register(ComponentCheck("fmt", "ensures the BUILD.yaml of a component is leeway fmt'ed", checkComponentsFmt)) 13 | } 14 | 15 | func checkComponentsFmt(comp *leeway.Component) ([]Finding, error) { 16 | fc, err := os.ReadFile(filepath.Join(comp.Origin, "BUILD.yaml")) 17 | if err != nil { 18 | return nil, err 19 | } 20 | if len(fc) == 0 { 21 | // empty BUILD.yaml files are ok 22 | return nil, nil 23 | } 24 | 25 | buf := bytes.NewBuffer(nil) 26 | err = leeway.FormatBUILDyaml(buf, bytes.NewReader(fc), false) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | if bytes.EqualFold(buf.Bytes(), fc) { 32 | return nil, nil 33 | } 34 | 35 | return []Finding{ 36 | { 37 | Component: comp, 38 | Description: "component's BUILD.yaml is not formated using `leeway fmt`", 39 | Error: false, 40 | }, 41 | }, nil 42 | } 43 | -------------------------------------------------------------------------------- /pkg/vet/docker.go: -------------------------------------------------------------------------------- 1 | package vet 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "regexp" 8 | "strings" 9 | 10 | log "github.com/sirupsen/logrus" 11 | 12 | "github.com/gitpod-io/leeway/pkg/leeway" 13 | ) 14 | 15 | func init() { 16 | register(PackageCheck("copy-from-pacakge", "attempts to find broken package paths in COPY and ADD statements", leeway.DockerPackage, checkDockerCopyFromPackage)) 17 | } 18 | 19 | var ( 20 | filesystemSafePathPattern = regexp.MustCompile(`([a-zA-Z0-9\.]+\-)+\-([a-zA-Z0-9\.\-]+)`) 21 | ) 22 | 23 | func checkDockerCopyFromPackage(pkg *leeway.Package) ([]Finding, error) { 24 | cfg, ok := pkg.Config.(leeway.DockerPkgConfig) 25 | if !ok { 26 | // this is an error as compared to a finding because the issue most likely is with leeway, 27 | // and not a user config error. 28 | return nil, fmt.Errorf("Docker package does not have docker package config") 29 | } 30 | 31 | var dockerfileFN string 32 | for _, src := range pkg.Sources { 33 | if strings.HasSuffix(src, "/"+cfg.Dockerfile) { 34 | dockerfileFN = src 35 | } 36 | } 37 | if dockerfileFN == "" { 38 | return []Finding{{ 39 | Component: pkg.C, 40 | Package: pkg, 41 | Description: "package has no Dockerfile", 42 | Error: true, 43 | }}, nil 44 | } 45 | 46 | f, err := os.Open(dockerfileFN) 47 | if err != nil { 48 | return nil, err 49 | } 50 | defer f.Close() 51 | 52 | var findings []Finding 53 | scanner := bufio.NewScanner(f) 54 | for scanner.Scan() { 55 | segs := strings.Fields(scanner.Text()) 56 | if len(segs) == 0 { 57 | continue 58 | } 59 | 60 | cmd := strings.ToLower(segs[0]) 61 | if cmd != "add" && cmd != "copy" { 62 | continue 63 | } 64 | 65 | for _, s := range segs[1 : len(segs)-1] { 66 | if !filesystemSafePathPattern.MatchString(s) { 67 | continue 68 | } 69 | 70 | pth := filesystemSafePathPattern.FindString(s) 71 | log.WithField("pth", pth).WithField("dockerFile", dockerfileFN).WithField("pkg", pkg.FullName()).Debug("found potential copy source path") 72 | 73 | // we've found something that looks like a path - check if we have a dependency that could satisfy it 74 | var satisfied bool 75 | for _, dep := range pkg.GetDependencies() { 76 | if pkg.BuildLayoutLocation(dep) == pth { 77 | satisfied = true 78 | break 79 | } 80 | } 81 | if satisfied { 82 | continue 83 | } 84 | 85 | findings = append(findings, Finding{ 86 | Description: fmt.Sprintf("%s copies from %s which looks like a package path, but no dependency satisfies it", cfg.Dockerfile, s), 87 | Component: pkg.C, 88 | Package: pkg, 89 | Error: false, 90 | }) 91 | } 92 | } 93 | if err := scanner.Err(); err != nil { 94 | return nil, err 95 | } 96 | 97 | return findings, nil 98 | } 99 | -------------------------------------------------------------------------------- /pkg/vet/docker_test.go: -------------------------------------------------------------------------------- 1 | package vet 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/google/go-cmp/cmp" 11 | 12 | "github.com/gitpod-io/leeway/pkg/leeway" 13 | ) 14 | 15 | func TestCheckDockerCopyFromPackage(t *testing.T) { 16 | tests := []struct { 17 | Name string 18 | Dockerfile string 19 | Deps []string 20 | Findings []string 21 | }{ 22 | { 23 | Name: "true positive copy", 24 | Dockerfile: `FROM alpine:latest 25 | COPY from-some-pkg--build/hello.txt hello.txt`, 26 | Findings: []string{ 27 | "Dockerfile copies from from-some-pkg--build/hello.txt which looks like a package path, but no dependency satisfies it", 28 | }, 29 | }, 30 | { 31 | Name: "true negative copy", 32 | Dockerfile: `FROM alpine:latest 33 | COPY from-some-pkg--build/hello.txt hello.txt`, 34 | Deps: []string{"from-some-pkg:build"}, 35 | }, 36 | { 37 | Name: "true positive add", 38 | Dockerfile: `FROM alpine:latest 39 | ADD from-some-pkg--build/hello.txt hello.txt`, 40 | Findings: []string{ 41 | "Dockerfile copies from from-some-pkg--build/hello.txt which looks like a package path, but no dependency satisfies it", 42 | }, 43 | }, 44 | { 45 | Name: "true negative add", 46 | Dockerfile: `FROM alpine:latest 47 | ADD from-some-pkg--build/hello.txt hello.txt`, 48 | Deps: []string{"from-some-pkg:build"}, 49 | }, 50 | } 51 | 52 | for _, test := range tests { 53 | t.Run(test.Name, func(t *testing.T) { 54 | failOnErr := func(err error) { 55 | if err != nil { 56 | t.Fatalf("cannot set up test: %q", err) 57 | } 58 | } 59 | 60 | tmpdir, err := os.MkdirTemp("", "leeway-test-*") 61 | failOnErr(err) 62 | defer os.RemoveAll(tmpdir) 63 | 64 | var pkgdeps string 65 | failOnErr(os.WriteFile(filepath.Join(tmpdir, "WORKSPACE.yaml"), []byte("environmentManifest:\n - name: \"docker\"\n command: [\"echo\"]"), 0644)) 66 | for _, dep := range test.Deps { 67 | segs := strings.Split(dep, ":") 68 | loc := filepath.Join(tmpdir, segs[0]) 69 | failOnErr(os.MkdirAll(loc, 0755)) 70 | failOnErr(os.WriteFile(filepath.Join(loc, "BUILD.yaml"), []byte(fmt.Sprintf(`packages: 71 | - name: %s 72 | type: generic`, segs[1])), 0755)) 73 | 74 | if pkgdeps == "" { 75 | pkgdeps = "\n deps:\n" 76 | } 77 | pkgdeps += fmt.Sprintf(" - %s\n", dep) 78 | } 79 | failOnErr(os.MkdirAll(filepath.Join(tmpdir, "test-pkg"), 0755)) 80 | failOnErr(os.WriteFile(filepath.Join(tmpdir, "test-pkg", "Dockerfile"), []byte(test.Dockerfile), 0644)) 81 | failOnErr(os.WriteFile(filepath.Join(tmpdir, "test-pkg", "BUILD.yaml"), []byte(fmt.Sprintf(`packages: 82 | - name: docker 83 | type: docker 84 | config: 85 | dockerfile: Dockerfile%s 86 | `, pkgdeps)), 0644)) 87 | 88 | ws, err := leeway.FindWorkspace(tmpdir, leeway.Arguments{}, "", "") 89 | failOnErr(err) 90 | pkg, ok := ws.Packages["test-pkg:docker"] 91 | if !ok { 92 | t.Fatalf("cannot find test package: test-pkg:docker") 93 | } 94 | 95 | findings, err := checkDockerCopyFromPackage(pkg) 96 | if err != nil { 97 | t.Fatalf("unexpected error: %s", err.Error()) 98 | } 99 | 100 | var fs []string 101 | if len(findings) > 0 { 102 | fs = make([]string, len(findings)) 103 | for i := range findings { 104 | fs[i] = findings[i].Description 105 | } 106 | } 107 | if diff := cmp.Diff(test.Findings, fs); diff != "" { 108 | t.Errorf("MakeGatewayInfo() mismatch (-want +got):\n%s", diff) 109 | } 110 | }) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /pkg/vet/generic.go: -------------------------------------------------------------------------------- 1 | package vet 2 | 3 | import ( 4 | "fmt" 5 | 6 | log "github.com/sirupsen/logrus" 7 | 8 | "github.com/gitpod-io/leeway/pkg/leeway" 9 | ) 10 | 11 | func init() { 12 | register(PackageCheck("use-package", "attempts to find broken package paths in the commands", leeway.GenericPackage, checkArgsReferingToPackage)) 13 | } 14 | 15 | func checkArgsReferingToPackage(pkg *leeway.Package) ([]Finding, error) { 16 | cfg, ok := pkg.Config.(leeway.GenericPkgConfig) 17 | if !ok { 18 | // this is an error as compared to a finding because the issue most likely is with leeway, 19 | // and not a user config error. 20 | return nil, fmt.Errorf("Generic package does not have generic package config") 21 | } 22 | 23 | checkForFindings := func(fs []Finding, segmentIndex int, seg string) (findings []Finding) { 24 | findings = fs 25 | if !filesystemSafePathPattern.MatchString(seg) { 26 | return findings 27 | } 28 | 29 | pth := filesystemSafePathPattern.FindString(seg) 30 | log.WithField("pth", pth).WithField("pkg", pkg.FullName()).Debug("found potential package use") 31 | 32 | // we've found something that looks like a path - check if we have a dependency that could satisfy it 33 | var satisfied bool 34 | for _, dep := range pkg.GetDependencies() { 35 | if pkg.BuildLayoutLocation(dep) == pth { 36 | satisfied = true 37 | break 38 | } 39 | } 40 | if satisfied { 41 | return findings 42 | } 43 | 44 | findings = append(findings, Finding{ 45 | Description: fmt.Sprintf("Command/Test %d refers to %s which looks like a package path, but no dependency satisfies it", segmentIndex, seg), 46 | Component: pkg.C, 47 | Package: pkg, 48 | Error: false, 49 | }) 50 | return findings 51 | } 52 | 53 | var findings []Finding 54 | for i, cmd := range cfg.Commands { 55 | for _, seg := range cmd { 56 | findings = checkForFindings(findings, i, seg) 57 | } 58 | } 59 | for i, cmd := range cfg.Test { 60 | for _, seg := range cmd { 61 | findings = checkForFindings(findings, i, seg) 62 | } 63 | } 64 | 65 | return findings, nil 66 | } 67 | -------------------------------------------------------------------------------- /pkg/vet/golang.go: -------------------------------------------------------------------------------- 1 | package vet 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/gitpod-io/leeway/pkg/leeway" 8 | ) 9 | 10 | func init() { 11 | register(PackageCheck("has-gomod", "ensures all Go packages have a go.mod file in their source list", leeway.GoPackage, checkGolangHasGomod)) 12 | register(PackageCheck("has-buildflags", "checks for use of deprecated buildFlags config", leeway.GoPackage, checkGolangHasBuildFlags)) 13 | } 14 | 15 | func checkGolangHasGomod(pkg *leeway.Package) ([]Finding, error) { 16 | var ( 17 | foundGoMod bool 18 | foundGoSum bool 19 | ) 20 | for _, src := range pkg.Sources { 21 | if strings.HasSuffix(src, "/go.mod") { 22 | foundGoMod = true 23 | } 24 | if strings.HasSuffix(src, "/go.sum") { 25 | foundGoSum = true 26 | } 27 | if foundGoSum && foundGoMod { 28 | return nil, nil 29 | } 30 | } 31 | 32 | var f []Finding 33 | if !foundGoMod { 34 | f = append(f, Finding{ 35 | Component: pkg.C, 36 | Description: "package sources contain no go.mod file", 37 | Error: true, 38 | Package: pkg, 39 | }) 40 | } 41 | if !foundGoSum { 42 | f = append(f, Finding{ 43 | Component: pkg.C, 44 | Description: "package sources contain no go.sum file", 45 | Error: true, 46 | Package: pkg, 47 | }) 48 | } 49 | return f, nil 50 | } 51 | 52 | func checkGolangHasBuildFlags(pkg *leeway.Package) ([]Finding, error) { 53 | goCfg, ok := pkg.Config.(leeway.GoPkgConfig) 54 | if !ok { 55 | return nil, fmt.Errorf("Go package does not have go package config") 56 | } 57 | 58 | if len(goCfg.BuildFlags) > 0 { 59 | return []Finding{{ 60 | Component: pkg.C, 61 | Description: "buildFlags are deprecated, use buildCommand instead", 62 | Error: false, 63 | Package: pkg, 64 | }}, nil 65 | } 66 | 67 | return nil, nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/vet/packages.go: -------------------------------------------------------------------------------- 1 | package vet 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/gitpod-io/leeway/pkg/leeway" 7 | ) 8 | 9 | func init() { 10 | register(PackageCheck("build-layout", "validates the build layout of all packages", "", checkBuildLayout)) 11 | } 12 | 13 | func checkBuildLayout(pkg *leeway.Package) (findings []Finding, err error) { 14 | layoutIdx := make(map[string]string) 15 | for dep, loc := range pkg.Layout { 16 | otherdep, taken := layoutIdx[loc] 17 | if !taken { 18 | layoutIdx[loc] = dep 19 | continue 20 | } 21 | 22 | findings = append(findings, Finding{ 23 | Description: fmt.Sprintf("build-time location %v is used by %v and %v", loc, dep, otherdep), 24 | Component: pkg.C, 25 | Error: true, 26 | Package: pkg, 27 | }) 28 | } 29 | return 30 | } 31 | -------------------------------------------------------------------------------- /pkg/vet/vet.go: -------------------------------------------------------------------------------- 1 | package vet 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "sort" 7 | 8 | log "github.com/sirupsen/logrus" 9 | "golang.org/x/xerrors" 10 | 11 | "github.com/gitpod-io/leeway/pkg/leeway" 12 | ) 13 | 14 | type checkFunc struct { 15 | info CheckInfo 16 | 17 | runPkg func(pkg *leeway.Package) ([]Finding, error) 18 | runCmp func(pkg *leeway.Component) ([]Finding, error) 19 | } 20 | 21 | func (cf *checkFunc) Info() CheckInfo { 22 | return cf.info 23 | } 24 | 25 | func (cf *checkFunc) Init(leeway.Workspace) error { 26 | return nil 27 | } 28 | 29 | func (cf *checkFunc) RunPkg(pkg *leeway.Package) ([]Finding, error) { 30 | if cf.runPkg == nil { 31 | return nil, xerrors.Errorf("not a package check") 32 | } 33 | return cf.runPkg(pkg) 34 | } 35 | 36 | func (cf *checkFunc) RunCmp(pkg *leeway.Component) ([]Finding, error) { 37 | if cf.runCmp == nil { 38 | return nil, xerrors.Errorf("has no component check") 39 | } 40 | return cf.runCmp(pkg) 41 | } 42 | 43 | // PackageCheck produces a new check for a leeway package 44 | func PackageCheck(name, desc string, tpe leeway.PackageType, chk func(pkg *leeway.Package) ([]Finding, error)) Check { 45 | return &checkFunc{ 46 | info: CheckInfo{ 47 | Name: fmt.Sprintf("%s:%s", tpe, name), 48 | Description: desc, 49 | AppliesToType: &tpe, 50 | PackageCheck: true, 51 | }, 52 | runPkg: chk, 53 | } 54 | } 55 | 56 | // ComponentCheck produces a new check for a leeway component 57 | func ComponentCheck(name, desc string, chk func(pkg *leeway.Component) ([]Finding, error)) Check { 58 | return &checkFunc{ 59 | info: CheckInfo{ 60 | Name: fmt.Sprintf("component:%s", name), 61 | Description: desc, 62 | PackageCheck: false, 63 | }, 64 | runCmp: chk, 65 | } 66 | } 67 | 68 | // Check implements a vet check 69 | type Check interface { 70 | Info() CheckInfo 71 | 72 | Init(ws leeway.Workspace) error 73 | RunPkg(pkg *leeway.Package) ([]Finding, error) 74 | RunCmp(pkg *leeway.Component) ([]Finding, error) 75 | } 76 | 77 | // CheckInfo describes a check 78 | type CheckInfo struct { 79 | Name string 80 | Description string 81 | PackageCheck bool 82 | AppliesToType *leeway.PackageType 83 | } 84 | 85 | // Finding describes a check finding. If the package is nil, the finding applies to the component 86 | type Finding struct { 87 | Check string 88 | Component *leeway.Component 89 | Package *leeway.Package 90 | Description string 91 | Error bool 92 | } 93 | 94 | // MarshalJSON marshals a finding to JSON 95 | func (f Finding) MarshalJSON() ([]byte, error) { 96 | var p struct { 97 | Check string `json:"check"` 98 | Component string `json:"component"` 99 | Package string `json:"package,omitempty"` 100 | Description string `json:"description,omitempty"` 101 | Error bool `json:"error"` 102 | } 103 | p.Check = f.Check 104 | p.Component = f.Component.Name 105 | if f.Package != nil { 106 | p.Package = f.Package.FullName() 107 | } 108 | p.Description = f.Description 109 | p.Error = f.Error 110 | 111 | return json.Marshal(p) 112 | } 113 | 114 | var _checks = make(map[string]Check) 115 | 116 | func register(c Check) { 117 | cn := c.Info().Name 118 | if _, exists := _checks[cn]; exists { 119 | panic(fmt.Sprintf("check %s is already registered", cn)) 120 | } 121 | _checks[cn] = c 122 | } 123 | 124 | // Checks returns a list of all available checks 125 | func Checks() []Check { 126 | l := make([]Check, 0, len(_checks)) 127 | for _, c := range _checks { 128 | l = append(l, c) 129 | } 130 | sort.Slice(l, func(i, j int) bool { return l[i].Info().Name < l[j].Info().Name }) 131 | return l 132 | } 133 | 134 | // RunOpt modifies the run behaviour 135 | type RunOpt func(*runOptions) 136 | 137 | type runOptions struct { 138 | Packages StringSet 139 | Components StringSet 140 | Checks []string 141 | } 142 | 143 | // StringSet identifies a string as part of a set 144 | type StringSet map[string]struct{} 145 | 146 | // OnPackages makes run check these packages only 147 | func OnPackages(n StringSet) RunOpt { 148 | return func(r *runOptions) { 149 | r.Packages = n 150 | } 151 | } 152 | 153 | // OnComponents makes run check these components only 154 | func OnComponents(n StringSet) RunOpt { 155 | return func(r *runOptions) { 156 | r.Components = n 157 | } 158 | } 159 | 160 | // WithChecks runs these checks only 161 | func WithChecks(n []string) RunOpt { 162 | return func(r *runOptions) { 163 | r.Checks = n 164 | } 165 | } 166 | 167 | // Run runs all checks on all packages 168 | func Run(workspace leeway.Workspace, options ...RunOpt) ([]Finding, []error) { 169 | var opts runOptions 170 | for _, o := range options { 171 | o(&opts) 172 | } 173 | 174 | var checks []Check 175 | if len(opts.Checks) == 0 { 176 | checks = make([]Check, 0, len(_checks)) 177 | for _, c := range _checks { 178 | checks = append(checks, c) 179 | } 180 | } else { 181 | log.WithField("checks", opts.Checks).Debug("running selected checks only") 182 | for _, cn := range opts.Checks { 183 | c, ok := _checks[cn] 184 | if !ok { 185 | return nil, []error{xerrors.Errorf("check %s not found", cn)} 186 | } 187 | checks = append(checks, c) 188 | } 189 | } 190 | for _, check := range checks { 191 | err := check.Init(workspace) 192 | if err != nil { 193 | return nil, []error{fmt.Errorf("init %s: %w", check.Info().Name, err)} 194 | } 195 | } 196 | 197 | var ( 198 | findings []Finding 199 | errs []error 200 | 201 | runCompCheck = func(c Check, comp *leeway.Component) { 202 | info := c.Info() 203 | if info.PackageCheck { 204 | return 205 | } 206 | 207 | log.WithField("check", info.Name).WithField("cmp", comp.Name).Debug("running component check") 208 | f, err := c.RunCmp(comp) 209 | if err != nil { 210 | errs = append(errs, fmt.Errorf("[%s] %s: %w", info.Name, comp.Name, err)) 211 | return 212 | } 213 | for i := range f { 214 | f[i].Check = info.Name 215 | } 216 | findings = append(findings, f...) 217 | } 218 | runPkgCheck = func(c Check, pkg *leeway.Package) { 219 | info := c.Info() 220 | if !info.PackageCheck { 221 | return 222 | } 223 | 224 | if info.AppliesToType != nil && *info.AppliesToType != pkg.Type { 225 | return 226 | } 227 | 228 | log.WithField("check", info.Name).WithField("pkg", pkg.FullName()).Debug("running package check") 229 | f, err := c.RunPkg(pkg) 230 | if err != nil { 231 | errs = append(errs, fmt.Errorf("[%s] %s: %w", info.Name, pkg.FullName(), err)) 232 | return 233 | } 234 | for i := range f { 235 | f[i].Check = info.Name 236 | } 237 | findings = append(findings, f...) 238 | } 239 | ) 240 | 241 | if len(opts.Components) > 0 { 242 | for n, comp := range workspace.Components { 243 | if _, ok := opts.Components[n]; !ok { 244 | continue 245 | } 246 | 247 | for _, check := range checks { 248 | runCompCheck(check, comp) 249 | } 250 | } 251 | } else if len(opts.Packages) > 0 { 252 | for n, pkg := range workspace.Packages { 253 | if _, ok := opts.Packages[n]; !ok { 254 | continue 255 | } 256 | 257 | for _, check := range checks { 258 | runPkgCheck(check, pkg) 259 | } 260 | } 261 | } else { 262 | for _, check := range checks { 263 | for _, comp := range workspace.Components { 264 | runCompCheck(check, comp) 265 | } 266 | 267 | for _, pkg := range workspace.Packages { 268 | runPkgCheck(check, pkg) 269 | } 270 | } 271 | } 272 | 273 | return findings, errs 274 | } 275 | -------------------------------------------------------------------------------- /pkg/vet/yarn.go: -------------------------------------------------------------------------------- 1 | package vet 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "regexp" 11 | "strings" 12 | 13 | log "github.com/sirupsen/logrus" 14 | "golang.org/x/xerrors" 15 | 16 | "github.com/gitpod-io/leeway/pkg/leeway" 17 | ) 18 | 19 | func init() { 20 | register(&checkImplicitTransitiveDependencies{}) 21 | register(PackageCheck("node-modules-source", "checks if the package's sources include node_modules/", leeway.YarnPackage, checkYarnNodeModulesSource)) 22 | } 23 | 24 | type pkgJSON struct { 25 | Name string `json:"name"` 26 | Dependencies map[string]interface{} `json:"dependencies"` 27 | } 28 | 29 | type checkImplicitTransitiveDependencies struct { 30 | pkgs map[string][]string 31 | } 32 | 33 | func (c *checkImplicitTransitiveDependencies) Info() CheckInfo { 34 | tpe := leeway.YarnPackage 35 | return CheckInfo{ 36 | Name: "yarn:implicit-transitive-dependency", 37 | Description: "checks if the package's code uses another Yarn package in the workspace without declaring the dependency", 38 | AppliesToType: &tpe, 39 | PackageCheck: true, 40 | } 41 | } 42 | 43 | func (c *checkImplicitTransitiveDependencies) Init(ws leeway.Workspace) error { 44 | c.pkgs = make(map[string][]string) 45 | for pn, p := range ws.Packages { 46 | if p.Type != leeway.YarnPackage { 47 | continue 48 | } 49 | 50 | pkgJSON, err := c.getPkgJSON(p) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | if pkgJSON.Name == "" { 56 | continue 57 | } 58 | c.pkgs[pkgJSON.Name] = append(c.pkgs[pkgJSON.Name], pn) 59 | } 60 | return nil 61 | } 62 | 63 | func (c *checkImplicitTransitiveDependencies) getPkgJSON(pkg *leeway.Package) (*pkgJSON, error) { 64 | var ( 65 | found bool 66 | pkgFN = filepath.Join(pkg.C.Origin, "package.json") 67 | ) 68 | for _, src := range pkg.Sources { 69 | if src == pkgFN { 70 | found = true 71 | break 72 | } 73 | } 74 | if !found { 75 | return nil, xerrors.Errorf("package %s has no package.json", pkg.FullName()) 76 | } 77 | 78 | fc, err := os.ReadFile(pkgFN) 79 | if err != nil { 80 | return nil, err 81 | } 82 | var res pkgJSON 83 | err = json.Unmarshal(fc, &res) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | if res.Name == "" { 89 | return nil, xerrors.Errorf("package %s has no Yarn package name", pkg.FullName()) 90 | } 91 | 92 | return &res, nil 93 | } 94 | 95 | func (c *checkImplicitTransitiveDependencies) grepInFile(fn string, pat *regexp.Regexp) (contains bool, err error) { 96 | f, err := os.Open(fn) 97 | if err != nil { 98 | return 99 | } 100 | defer f.Close() 101 | 102 | r := bufio.NewReader(f) 103 | for { 104 | bt, err := r.ReadBytes('\n') 105 | if err == io.EOF { 106 | break 107 | } 108 | if err != nil { 109 | return false, err 110 | } 111 | 112 | if pat.Match(bt) { 113 | return true, nil 114 | } 115 | } 116 | 117 | return false, nil 118 | } 119 | 120 | func (c *checkImplicitTransitiveDependencies) RunCmp(pkg *leeway.Component) ([]Finding, error) { 121 | return nil, fmt.Errorf("not a component check") 122 | } 123 | 124 | func (c *checkImplicitTransitiveDependencies) RunPkg(pkg *leeway.Package) ([]Finding, error) { 125 | depsInCode := make(map[string]string) 126 | for _, src := range pkg.Sources { 127 | switch filepath.Ext(src) { 128 | case ".js": 129 | case ".ts": 130 | default: 131 | continue 132 | } 133 | 134 | for yarnpkg := range c.pkgs { 135 | r, _ := regexp.Compile(fmt.Sprintf("['\"]%s['\"/]", yarnpkg)) 136 | ok, err := c.grepInFile(src, r) 137 | if err != nil { 138 | return nil, err 139 | } 140 | if ok { 141 | depsInCode[yarnpkg] = src 142 | } 143 | } 144 | } 145 | 146 | var findings []Finding 147 | for yarnDep, src := range depsInCode { 148 | var found bool 149 | for _, leewayDep := range c.pkgs[yarnDep] { 150 | for _, dep := range pkg.GetDependencies() { 151 | if dep.FullName() == leewayDep { 152 | found = true 153 | break 154 | } 155 | } 156 | } 157 | if found { 158 | continue 159 | } 160 | 161 | findings = append(findings, Finding{ 162 | Description: fmt.Sprintf("%s depends on the workspace Yarn-package %s (provided by %s) but does not declare that dependency in its BUILD.yaml", src, yarnDep, strings.Join(c.pkgs[yarnDep], ", ")), 163 | Error: true, 164 | Component: pkg.C, 165 | Package: pkg, 166 | }) 167 | } 168 | 169 | pkgjson, err := c.getPkgJSON(pkg) 170 | if err != nil { 171 | return findings, err 172 | } 173 | for yarnDep, src := range depsInCode { 174 | _, found := pkgjson.Dependencies[yarnDep] 175 | if found { 176 | continue 177 | } 178 | 179 | log.WithField("pkg", pkg.FullName()).WithField("pkgJsonDeclaredDeps", pkgjson.Dependencies).WithField("yarnName", pkgjson.Name).Debug("found use of implicit transitive dependency") 180 | findings = append(findings, Finding{ 181 | Description: fmt.Sprintf("%s depends on the workspace Yarn-package %s but does not declare that dependency in its package.json", src, yarnDep), 182 | Component: pkg.C, 183 | Package: pkg, 184 | }) 185 | } 186 | 187 | return findings, nil 188 | } 189 | 190 | func checkYarnNodeModulesSource(pkg *leeway.Package) ([]Finding, error) { 191 | var res []string 192 | for _, src := range pkg.Sources { 193 | segs := strings.Split(src, "/") 194 | var found bool 195 | for _, seg := range segs { 196 | if seg == "node_modules" { 197 | found = true 198 | break 199 | } 200 | } 201 | if !found { 202 | continue 203 | } 204 | if res == nil || len(segs) < len(res) { 205 | res = segs 206 | } 207 | } 208 | 209 | if res == nil { 210 | return nil, nil 211 | } 212 | return []Finding{ 213 | { 214 | Component: pkg.C, 215 | Package: pkg, 216 | Description: fmt.Sprintf("package contains node_modules/ as source: %s", strings.Join(res, "/")), 217 | }, 218 | }, nil 219 | } 220 | --------------------------------------------------------------------------------