├── .github ├── depandabot.yml └── workflows │ ├── main.yml │ └── release.yml ├── .golangci.yml ├── .goreleaser.yml ├── Dockerfile ├── LICENSE ├── README.md ├── cmd └── dot │ ├── root.go │ └── version.go ├── dot.yml ├── go.mod ├── go.sum ├── images └── demo.gif ├── main.go └── pkg ├── artifacts └── dockermanager.go ├── models └── models.go ├── runner ├── docker.go └── docker_test.go ├── store ├── memorystore.go └── memorystore_test.go └── utils ├── compress.go └── logger.go /.github/depandabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | 8 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Dot Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - "master" 7 | - "feature/**" 8 | - "security/**" 9 | - "fix/**" 10 | - "release/**" 11 | pull_request: 12 | branches: [ "master" ] 13 | 14 | jobs: 15 | 16 | build: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v3 20 | 21 | - name: Pull Alpine image 22 | run: docker pull alpine:latest 23 | 24 | - name: Set up Go 25 | uses: actions/setup-go@v4 26 | with: 27 | go-version: '1.21.1' 28 | 29 | - name: Run tests 30 | run: go test -coverprofile=coverage.out ./... 31 | 32 | - name: Run codacy-coverage-reporter 33 | uses: codacy/codacy-coverage-reporter-action@v1 34 | with: 35 | project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} 36 | coverage-reports: coverage.out 37 | language: go 38 | force-coverage-parser: go 39 | 40 | - name: Run Dot 41 | run: go run main.go -m 42 | 43 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Dot Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - "**" 7 | 8 | env: 9 | REGISTRY: ghcr.io 10 | IMAGE_NAME: ${{ github.repository }} 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | permissions: 16 | contents: write 17 | packages: write 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | 22 | - name: Set up Go 23 | uses: actions/setup-go@v4 24 | with: 25 | go-version: '1.21.1' 26 | 27 | - name: Run GoReleaser 28 | uses: goreleaser/goreleaser-action@v5 29 | with: 30 | distribution: goreleaser 31 | version: latest 32 | args: release --clean 33 | env: 34 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 35 | 36 | - name: Set up QEMU 37 | uses: docker/setup-qemu-action@v3 38 | 39 | - name: Set up Docker Buildx 40 | uses: docker/setup-buildx-action@v3 41 | 42 | - name: Login to Docker Hub 43 | uses: docker/login-action@v3 44 | with: 45 | registry: ${{ env.REGISTRY }} 46 | username: ${{ github.actor }} 47 | password: ${{ secrets.GITHUB_TOKEN }} 48 | 49 | - name: Build and push 50 | uses: docker/build-push-action@v5 51 | with: 52 | context: . 53 | push: true 54 | build-args: | 55 | VERSION=${{ github.ref_name }} 56 | BUILDDATE=${{ github.event.repository.updated_at }} 57 | COMMIT=${{ github.sha }} 58 | tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest 59 | 60 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | disable-all: true 3 | enable: 4 | - staticcheck 5 | - errcheck 6 | - govet 7 | - unused -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod tidy 4 | builds: 5 | - binary: dot 6 | env: 7 | - CGO_ENABLED=0 8 | goos: 9 | - linux 10 | goarch: 11 | - amd64 12 | - arm64 13 | flags: 14 | - -buildvcs=false 15 | ldflags: 16 | - "-s -w -X 'github.com/opnlabs/dot/cmd/dot.version={{ .Version }}' -X 'github.com/opnlabs/dot/cmd/dot.builddate={{ .Date }}' -X 'github.com/opnlabs/dot/cmd/dot.commit={{ .Commit }}'" 17 | archives: 18 | - name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" 19 | checksum: 20 | name_template: 'checksums.txt' 21 | snapshot: 22 | name_template: "{{ incpatch .Version }}-alpha" 23 | changelog: 24 | sort: asc 25 | filters: 26 | exclude: 27 | - '^docs:' 28 | - '^test:' -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.21-alpine3.17 as Builder 2 | ARG VERSION 3 | ARG BUILDDATE 4 | ARG COMMIT 5 | WORKDIR /app 6 | COPY . . 7 | RUN go build -o dot -ldflags="-X 'github.com/opnlabs/dot/cmd/dot.version=$VERSION' \ 8 | -X 'github.com/opnlabs/dot/cmd/dot.builddate=$BUILDDATE' \ 9 | -X 'github.com/opnlabs/dot/cmd/dot.commit=$COMMIT'" 10 | 11 | FROM alpine:3.18.2 12 | COPY --from=Builder /app/dot /usr/bin/dot 13 | WORKDIR /app 14 | ENTRYPOINT ["/usr/bin/dot"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Hariharan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dot 2 | [![Go Reference](https://pkg.go.dev/badge/github.com/opnlabs/dot.svg)](https://pkg.go.dev/github.com/opnlabs/dot) [![Dot](https://github.com/opnlabs/dot/actions/workflows/main.yml/badge.svg)](https://github.com/opnlabs/dot/actions/workflows/main.yml) [![Go Report Card](https://goreportcard.com/badge/github.com/cvhariharan/dot)](https://goreportcard.com/report/github.com/cvhariharan/dot) [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/1a5800bae3c143c29e3559e3f46bffb1)](https://app.codacy.com/gh/opnlabs/dot/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_coverage) 3 | 4 | A minimal CI. Designed to be local first. 5 | 6 | All the jobs run inside docker containers. `Dot` communicates with the Docker daemon using the [Docker client API](https://pkg.go.dev/github.com/docker/docker/client#section-readme). 7 | 8 | Refer the project [wiki](https://github.com/opnlabs/dot/wiki) to learn more about dot. 9 | 10 |

11 | 12 |

13 | 14 | ## Features 15 | - Single binary, can run anywhere, on your machine or CI/CD systems 16 | - Multi stage builds with support for build artifacts 17 | - Simple yaml job definition 18 | - Bring your own Docker images. Supports private registries 19 | - Uses plain Docker 20 | - Supports conditional evaluation using expressions 21 | 22 | ## Installation 23 | Get the latest version from the [releases](https://github.com/opnlabs/dot/releases) section. 24 | 25 | The latest version can also be installed using `go`. 26 | ```bash 27 | go install github.com/opnlabs/dot@latest 28 | ``` 29 | 30 | ### Run using Docker 31 | ```bash 32 | docker run -it -v /var/run/docker.sock:/var/run/docker.sock -v /path/to/project:/app ghcr.io/opnlabs/dot:latest -m 33 | ``` 34 | 35 | ## Example 36 | This example uses [GoReleaser](https://github.com/goreleaser/goreleaser) to build this project. 37 | ```yaml 38 | stages: 39 | - test 40 | - security 41 | - build 42 | 43 | jobs: 44 | - name: Run tests 45 | stage: test 46 | image: "docker.io/golang:1.21.3" 47 | variables: 48 | - TEST: true 49 | script: 50 | - go test ./... 51 | condition: TEST 52 | 53 | - name: Run checks 54 | stage: security 55 | image: "docker.io/golangci/golangci-lint:latest" 56 | script: 57 | - golangci-lint run ./... 58 | 59 | - name: Build using Goreleaser 60 | stage: build 61 | image: "docker.io/golang:1.21.3-bookworm" 62 | script: 63 | - git config --global safe.directory '*' 64 | - curl -sfL https://goreleaser.com/static/run | bash -s -- build --snapshot 65 | artifacts: 66 | - dist 67 | ``` 68 | Extract the binary once the build is complete. 69 | ``` 70 | tar xvf .artifacts/artifacts-*.tar 71 | dist/dot_linux_amd64_v1/dot version 72 | ``` 73 | ### Build Dot with Dot 74 | This project can be built with `Dot`. The [dot.yml](dot.yml) file describes all the jobs necessary to build a linux binary. Clone the repo and run 75 | 76 | ```bash 77 | go run main.go -m 78 | ``` 79 | This should create an artifact tar file in the `.artifacts` directory with the linux binary `dot`. 80 | The `-m` flag gives `dot` access to the host's docker socket. This is required only if containers are created within `dot`. 81 |

82 | 83 |

84 | 85 | -------------------------------------------------------------------------------- /cmd/dot/root.go: -------------------------------------------------------------------------------- 1 | package dot 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | 11 | "github.com/expr-lang/expr" 12 | "github.com/go-playground/validator/v10" 13 | "github.com/opnlabs/dot/pkg/artifacts" 14 | "github.com/opnlabs/dot/pkg/models" 15 | "github.com/opnlabs/dot/pkg/runner" 16 | "github.com/opnlabs/dot/pkg/utils" 17 | "github.com/spf13/cobra" 18 | "golang.org/x/sync/errgroup" 19 | "gopkg.in/yaml.v3" 20 | ) 21 | 22 | var ( 23 | jobFilePath string 24 | mountDockerSocket bool 25 | envVars []string 26 | environmentVariables []models.Variable = make([]models.Variable, 0) 27 | username string 28 | password string 29 | validate *validator.Validate = validator.New(validator.WithRequiredStructEnabled()) 30 | ) 31 | 32 | var rootCmd = &cobra.Command{ 33 | Use: "dot", 34 | Short: "Dot is a minimal CI", 35 | Long: `Dot is a minimal CI that runs jobs defined in a file ( default dot.yml ) 36 | inside docker containers. Jobs can be divided into stages where jobs within a stage are executed 37 | concurrently.`, 38 | 39 | Run: func(cmd *cobra.Command, args []string) { 40 | 41 | if len(envVars) > 0 { 42 | for _, v := range envVars { 43 | variables := strings.Split(v, "=") 44 | if len(variables) != 2 { 45 | log.Fatalf("variables should be defined as KEY=VALUE: %s", v) 46 | } 47 | 48 | m := make(map[string]any) 49 | m[variables[0]] = variables[1] 50 | environmentVariables = append(environmentVariables, m) 51 | } 52 | } 53 | 54 | run() 55 | }, 56 | } 57 | 58 | func init() { 59 | rootCmd.Flags().StringVarP(&jobFilePath, "job-file-path", "f", "dot.yml", "Path to the job file.") 60 | rootCmd.Flags().BoolVarP(&mountDockerSocket, "mount-docker-socket", "m", false, "Mount docker socket. Required to run containers from dot.") 61 | rootCmd.Flags().StringVarP(&username, "registry-username", "u", "", "Username for the container registry") 62 | rootCmd.Flags().StringVarP(&password, "registry-password", "p", "", "Password / Token for the container registry") 63 | 64 | rootCmd.Flags().StringArrayVarP(&envVars, "environment-variable", "e", make([]string, 0), "Environment variables. KEY=VALUE") 65 | 66 | rootCmd.AddCommand(versionCmd) 67 | } 68 | 69 | // Execute runs the root command for dot. 70 | func Execute() { 71 | if err := rootCmd.Execute(); err != nil { 72 | log.Fatal(err) 73 | } 74 | } 75 | 76 | func run() { 77 | ctx := context.Background() 78 | contents, err := os.ReadFile(filepath.Clean(jobFilePath)) 79 | if err != nil { 80 | log.Fatal(err) 81 | } 82 | 83 | var jobFile models.JobFile 84 | err = yaml.Unmarshal(contents, &jobFile) 85 | if err != nil { 86 | log.Fatal(err) 87 | } 88 | 89 | err = validate.Struct(jobFile) 90 | if err != nil { 91 | log.Fatalf("Err(s):\n%+v\n", err) 92 | } 93 | 94 | stageMap := make(map[models.Stage][]models.Job) 95 | for _, v := range jobFile.Stages { 96 | stageMap[v] = make([]models.Job, 0) 97 | } 98 | 99 | for _, v := range jobFile.Jobs { 100 | if _, ok := stageMap[v.Stage]; !ok { 101 | log.Fatalf("stage not defined: %s", v.Stage) 102 | } 103 | 104 | // Create expr program with the variables passed as env 105 | if len(v.Condition) == 0 { 106 | v.Condition = `true` 107 | } 108 | 109 | env := make(map[string]any) 110 | for _, entries := range v.Variables { 111 | if len(entries) > 1 { 112 | log.Fatal("variables should be defined as a key value pair") 113 | } 114 | for k, value := range entries { 115 | env[k] = value 116 | } 117 | } 118 | 119 | p, err := expr.Compile(v.Condition, expr.Env(env), expr.AsBool()) 120 | if err != nil { 121 | log.Fatalf("condition evaluation failed for job %s: %v", v.Name, err) 122 | } 123 | output, err := expr.Run(p, env) 124 | if err != nil { 125 | log.Fatalf("condition evaluation failed for job %s: %v", v.Name, err) 126 | } 127 | 128 | // Only append to stageMap if the condition evaluates to true 129 | if output.(bool) { 130 | stageMap[v.Stage] = append(stageMap[v.Stage], v) 131 | } 132 | 133 | } 134 | 135 | dockerArtifactManager := artifacts.NewDockerArtifactsManager(".artifacts") 136 | 137 | for _, v := range jobFile.Stages { 138 | var eg errgroup.Group 139 | for _, job := range stageMap[v] { 140 | jobCtx, cancel := context.WithTimeout(ctx, time.Hour) 141 | defer cancel() 142 | 143 | func(job models.Job) { 144 | eg.Go(func() error { 145 | return runner.NewDockerRunner(job.Name, dockerArtifactManager, 146 | runner.DockerRunnerOptions{ 147 | ShowImagePull: true, 148 | Stdout: utils.NewColorLogger(job.Name, os.Stdout, true), 149 | Stderr: utils.NewColorLogger(job.Name, os.Stderr, false), 150 | MountDockerSocket: mountDockerSocket}). 151 | WithImage(job.Image). 152 | WithSrc(job.Src). 153 | WithCmd(job.Script). 154 | WithEntrypoint(job.Entrypoint). 155 | WithEnv(append(job.Variables, environmentVariables...)). 156 | WithCredentials(username, password). 157 | CreatesArtifacts(job.Artifacts).Run(jobCtx) 158 | }) 159 | }(job) 160 | } 161 | err := eg.Wait() 162 | if err != nil { 163 | log.Fatal(err) 164 | } 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /cmd/dot/version.go: -------------------------------------------------------------------------------- 1 | package dot 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | var ( 10 | version = "nightly" 11 | builddate = "unknown" 12 | commit = "unknown" 13 | ) 14 | 15 | var versionCmd = &cobra.Command{ 16 | Use: "version", 17 | Short: "Shows the current version of Dot CLI", 18 | Run: func(cmd *cobra.Command, args []string) { 19 | fmt.Println("Version:", version) 20 | fmt.Println("Build Date:", builddate) 21 | fmt.Println("Commit:", commit) 22 | }, 23 | } 24 | -------------------------------------------------------------------------------- /dot.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - test 3 | - security 4 | - build 5 | 6 | jobs: 7 | - name: Run tests 8 | stage: test 9 | image: "docker.io/golang:1.21.3" 10 | variables: 11 | - TEST: false 12 | script: 13 | - go test ./... 14 | condition: TEST 15 | 16 | - name: Run checks 17 | stage: security 18 | image: "docker.io/golangci/golangci-lint:latest" 19 | script: 20 | - golangci-lint run ./... 21 | 22 | # - name: Run GoSec 23 | # stage: security 24 | # image: "docker.io/securego/gosec:latest" 25 | # entrypoint: ["/bin/sh", "-c"] 26 | # script: 27 | # - gosec --help 28 | 29 | # - name: Build using Goreleaser 30 | # stage: build 31 | # image: "docker.io/golang:1.21.3-bookworm" 32 | # script: 33 | # - git config --global safe.directory '*' 34 | # - curl -sfL https://goreleaser.com/static/run | bash -s -- build --snapshot 35 | # artifacts: 36 | # - dist 37 | 38 | - name: Build job linux 39 | stage: build 40 | image: "docker.io/golang:1.21.3-bookworm" 41 | script: 42 | - git config --global safe.directory '*' 43 | - export VERSION=$(git describe --always) 44 | - export BUILDDATE=$(date) 45 | - export COMMIT=$(git log --format="%H" -n 1) 46 | - echo $BUILDDATE 47 | - | 48 | go build -o dot \ 49 | -ldflags="-X 'github.com/opnlabs/dot/cmd/dot.version=$VERSION' \ 50 | -X 'github.com/opnlabs/dot/cmd/dot.builddate=$BUILDDATE' \ 51 | -X 'github.com/opnlabs/dot/cmd/dot.commit=$COMMIT'" \ 52 | main.go 53 | artifacts: 54 | - dot 55 | 56 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/opnlabs/dot 2 | 3 | go 1.21.3 4 | 5 | require ( 6 | github.com/docker/docker v24.0.9+incompatible 7 | github.com/expr-lang/expr v1.15.7 8 | github.com/fatih/color v1.15.0 9 | github.com/go-playground/validator/v10 v10.16.0 10 | github.com/gosimple/slug v1.13.1 11 | github.com/rs/xid v1.5.0 12 | github.com/spf13/cobra v1.8.0 13 | github.com/stretchr/testify v1.8.4 14 | golang.org/x/sync v0.5.0 15 | gopkg.in/yaml.v3 v3.0.1 16 | ) 17 | 18 | require ( 19 | github.com/Microsoft/go-winio v0.6.1 // indirect 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/distribution/reference v0.5.0 // indirect 22 | github.com/docker/distribution v2.8.3+incompatible // indirect 23 | github.com/docker/go-connections v0.4.0 // indirect 24 | github.com/docker/go-units v0.5.0 // indirect 25 | github.com/gabriel-vasile/mimetype v1.4.2 // indirect 26 | github.com/go-playground/locales v0.14.1 // indirect 27 | github.com/go-playground/universal-translator v0.18.1 // indirect 28 | github.com/gogo/protobuf v1.3.2 // indirect 29 | github.com/gosimple/unidecode v1.0.1 // indirect 30 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 31 | github.com/leodido/go-urn v1.2.4 // indirect 32 | github.com/mattn/go-colorable v0.1.13 // indirect 33 | github.com/mattn/go-isatty v0.0.17 // indirect 34 | github.com/moby/term v0.5.0 // indirect 35 | github.com/morikuni/aec v1.0.0 // indirect 36 | github.com/opencontainers/go-digest v1.0.0 // indirect 37 | github.com/opencontainers/image-spec v1.0.2 // indirect 38 | github.com/pkg/errors v0.9.1 // indirect 39 | github.com/pmezard/go-difflib v1.0.0 // indirect 40 | github.com/spf13/pflag v1.0.5 // indirect 41 | golang.org/x/crypto v0.21.0 // indirect 42 | golang.org/x/mod v0.8.0 // indirect 43 | golang.org/x/net v0.23.0 // indirect 44 | golang.org/x/sys v0.18.0 // indirect 45 | golang.org/x/text v0.14.0 // indirect 46 | golang.org/x/time v0.4.0 // indirect 47 | golang.org/x/tools v0.6.0 // indirect 48 | gotest.tools/v3 v3.5.1 // indirect 49 | ) 50 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= 2 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= 3 | github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= 4 | github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= 5 | github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= 10 | github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= 11 | github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= 12 | github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 13 | github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= 14 | github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 15 | github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= 16 | github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= 17 | github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= 18 | github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 19 | github.com/expr-lang/expr v1.15.7 h1:BK0JcWUkoW6nrbLBo6xCKhz4BvH5DSOOu1Gx5lucyZo= 20 | github.com/expr-lang/expr v1.15.7/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ= 21 | github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= 22 | github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= 23 | github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= 24 | github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= 25 | github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= 26 | github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 27 | github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= 28 | github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= 29 | github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= 30 | github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= 31 | github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE= 32 | github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= 33 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 34 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 35 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 36 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 37 | github.com/gosimple/slug v1.13.1 h1:bQ+kpX9Qa6tHRaK+fZR0A0M2Kd7Pa5eHPPsb1JpHD+Q= 38 | github.com/gosimple/slug v1.13.1/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= 39 | github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= 40 | github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= 41 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 42 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 43 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 44 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 45 | github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= 46 | github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= 47 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 48 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 49 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 50 | github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= 51 | github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 52 | github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= 53 | github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= 54 | github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= 55 | github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= 56 | github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= 57 | github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= 58 | github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= 59 | github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= 60 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 61 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 62 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 63 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 64 | github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= 65 | github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 66 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 67 | github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= 68 | github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 69 | github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= 70 | github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= 71 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 72 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 73 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 74 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 75 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 76 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 77 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 78 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 79 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 80 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 81 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 82 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 83 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 84 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 85 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 86 | golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= 87 | golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= 88 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 89 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 90 | golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= 91 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 92 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 93 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 94 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 95 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 96 | golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= 97 | golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= 98 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 99 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 100 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 101 | golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= 102 | golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 103 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 104 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 105 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 106 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 107 | golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= 108 | golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 109 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 110 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 111 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 112 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 113 | golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= 114 | golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 115 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 116 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 117 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 118 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 119 | golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= 120 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 121 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 122 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 123 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 124 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 125 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 126 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 127 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 128 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 129 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 130 | gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= 131 | gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= 132 | -------------------------------------------------------------------------------- /images/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opnlabs/dot/67cf6f958fb80faa947ec9da3c35e0d3cdb45a23/images/demo.gif -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Dot is a local first CI system. 2 | // 3 | // Dot uses Docker to run jobs concurrently in stages. More info can be found at https://github.com/opnlabs/dot/wiki 4 | package main 5 | 6 | import ( 7 | "github.com/opnlabs/dot/cmd/dot" 8 | ) 9 | 10 | func main() { 11 | dot.Execute() 12 | } 13 | -------------------------------------------------------------------------------- /pkg/artifacts/dockermanager.go: -------------------------------------------------------------------------------- 1 | package artifacts 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/fs" 8 | "log" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/docker/docker/api/types" 14 | "github.com/docker/docker/client" 15 | "github.com/opnlabs/dot/pkg/store" 16 | ) 17 | 18 | type ArtifactManager interface { 19 | PublishArtifact(jobID, path string) (key string, err error) 20 | RetrieveArtifact(jobID string, keys []string) error 21 | } 22 | 23 | type DockerArtifactsManager struct { 24 | cli *client.Client 25 | artifactStore store.Store 26 | artifactsDir string 27 | } 28 | 29 | func NewDockerArtifactsManager(artifactsDir string) ArtifactManager { 30 | // Clear previous artifacts and create a new directory 31 | if _, err := os.Stat(artifactsDir); err == nil { 32 | if err := os.RemoveAll(artifactsDir); err != nil { 33 | log.Fatalf("could not remove %s directory: %v", artifactsDir, err) 34 | } 35 | } 36 | 37 | if err := os.Mkdir(artifactsDir, 0755); err != nil { 38 | log.Fatalf("could not create %s directory: %v", artifactsDir, err) 39 | } 40 | 41 | cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) 42 | if err != nil { 43 | log.Fatal(err) 44 | } 45 | 46 | return &DockerArtifactsManager{ 47 | cli: cli, 48 | artifactStore: store.NewMemStore(), 49 | artifactsDir: artifactsDir, 50 | } 51 | } 52 | 53 | // PublishArtifact takes in a jobID and path inside the job and moves the artifact to the artifact store and returns a key 54 | // that references the artifact. 55 | func (d *DockerArtifactsManager) PublishArtifact(jobID, path string) (string, error) { 56 | ctx := context.Background() 57 | r, _, err := d.cli.CopyFromContainer(ctx, jobID, path) 58 | if err != nil { 59 | return "", fmt.Errorf("could not copy artifact %s from container %s: %v", path, jobID, err) 60 | } 61 | 62 | f, err := os.CreateTemp(d.artifactsDir, "artifacts-*.tar") 63 | if err != nil { 64 | return "", fmt.Errorf("could not create artifacts tar file: %v", err) 65 | } 66 | 67 | if _, err := io.Copy(f, r); err != nil { 68 | return "", fmt.Errorf("could not copy file contents from container %s to artifact tar: %v", jobID, err) 69 | } 70 | 71 | _, fname := filepath.Split(f.Name()) 72 | return fname, d.artifactStore.Set(strings.TrimSpace(fname), filepath.Dir(path)) 73 | } 74 | 75 | // RetrieveArtifact takes in a jobID, keys slice and moves the artifact to the original path inside the job. 76 | // If the keys is nil, all artifacts will be moved into the job. 77 | // The original path is the path from where the artifact was pushed in PublishArtifact. 78 | func (d *DockerArtifactsManager) RetrieveArtifact(jobID string, keys []string) error { 79 | ctx := context.Background() 80 | 81 | if len(keys) > 0 { 82 | for _, v := range keys { 83 | originalPath, err := d.artifactStore.Get(strings.TrimSpace(v)) 84 | if err != nil { 85 | return fmt.Errorf("could not find original path for artifact %s: %v", v, err) 86 | } 87 | f, err := os.Open(filepath.Clean(v)) 88 | if err != nil { 89 | return fmt.Errorf("could not open artifact %s: %v", v, err) 90 | } 91 | defer f.Close() 92 | 93 | if err := d.cli.CopyToContainer(ctx, jobID, originalPath.(string), f, types.CopyToContainerOptions{}); err != nil { 94 | return fmt.Errorf("could not copy artifact %s to container %s: %v", v, jobID, err) 95 | } 96 | } 97 | } 98 | 99 | return filepath.Walk(d.artifactsDir, func(path string, info fs.FileInfo, err error) error { 100 | if err != nil { 101 | return err 102 | } 103 | 104 | if !strings.Contains(path, ".tar") { 105 | return nil 106 | } 107 | 108 | f, err := os.Open(path) 109 | if err != nil { 110 | return fmt.Errorf("could not open %s artifact for copying to container %s: %v", path, jobID, err) 111 | } 112 | defer f.Close() 113 | 114 | _, fname := filepath.Split(path) 115 | originalPath, err := d.artifactStore.Get(strings.TrimSpace(fname)) 116 | if err != nil { 117 | return fmt.Errorf("could not get %s from artifact store: %v", fname, err) 118 | } 119 | 120 | return d.cli.CopyToContainer(context.Background(), jobID, originalPath.(string), f, types.CopyToContainerOptions{}) 121 | }) 122 | } 123 | -------------------------------------------------------------------------------- /pkg/models/models.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Stage string 4 | 5 | // Variable represents a job variable as a key-value pair. 6 | // Variables are defined as an array of key-value pairs, so each Variable map only has 1 entry. 7 | type Variable map[string]any 8 | 9 | // JobFile represents the dot.yml file 10 | type JobFile struct { 11 | Stages []Stage `yaml:"stages" validate:"required,dive"` 12 | Jobs []Job `yaml:"jobs" validate:"required,dive"` 13 | } 14 | 15 | // Job represents a single job in a stage 16 | type Job struct { 17 | Name string `yaml:"name" validate:"required"` 18 | Src string `yaml:"src"` 19 | Stage Stage `yaml:"stage" validate:"required"` 20 | Variables []Variable `yaml:"variables"` 21 | Image string `yaml:"image" validate:"required"` 22 | Script []string `yaml:"script"` 23 | Entrypoint []string `yaml:"entrypoint"` 24 | Artifacts []string `yaml:"artifacts"` 25 | Condition string `yaml:"condition"` 26 | } 27 | -------------------------------------------------------------------------------- /pkg/runner/docker.go: -------------------------------------------------------------------------------- 1 | // Package runner implements the backend that executes the jobs. 2 | // 3 | // Docker runner uses the docker runtime to execute jobs. 4 | package runner 5 | 6 | import ( 7 | "context" 8 | "encoding/base64" 9 | "encoding/json" 10 | "fmt" 11 | "io" 12 | "log" 13 | "os" 14 | "path/filepath" 15 | "strings" 16 | 17 | "github.com/docker/docker/api/types" 18 | "github.com/docker/docker/api/types/container" 19 | "github.com/docker/docker/api/types/mount" 20 | "github.com/docker/docker/api/types/registry" 21 | "github.com/docker/docker/client" 22 | "github.com/docker/docker/pkg/stdcopy" 23 | "github.com/gosimple/slug" 24 | "github.com/opnlabs/dot/pkg/artifacts" 25 | "github.com/opnlabs/dot/pkg/models" 26 | "github.com/opnlabs/dot/pkg/utils" 27 | "github.com/rs/xid" 28 | ) 29 | 30 | const ( 31 | ARTIFACTS_DIR = ".artifacts" 32 | WORKING_DIR = "/app" 33 | ) 34 | 35 | type DockerRunnerOptions struct { 36 | ShowImagePull bool 37 | Stdout io.Writer 38 | Stderr io.Writer 39 | MountDockerSocket bool 40 | } 41 | 42 | type DockerRunner struct { 43 | name string 44 | image string 45 | src string 46 | env []string 47 | cmd []string 48 | entrypoint []string 49 | containerID string 50 | workingDirectory string 51 | artifacts []string 52 | artifactManager artifacts.ArtifactManager 53 | dockerOptions DockerRunnerOptions 54 | authConfig string 55 | } 56 | 57 | func NewDockerRunner(name string, artifactManager artifacts.ArtifactManager, dockerOptions DockerRunnerOptions) *DockerRunner { 58 | wd, err := os.Getwd() 59 | if err != nil { 60 | log.Fatal(err) 61 | } 62 | jobName := slug.Make(fmt.Sprintf("%s-%s", name, xid.New().String())) 63 | 64 | if dockerOptions.Stdout == nil { 65 | dockerOptions.Stdout = os.Stdout 66 | } 67 | if dockerOptions.Stderr == nil { 68 | dockerOptions.Stderr = os.Stderr 69 | } 70 | 71 | return &DockerRunner{ 72 | name: jobName, 73 | src: filepath.Clean(""), 74 | workingDirectory: wd, 75 | artifactManager: artifactManager, 76 | dockerOptions: dockerOptions, 77 | } 78 | } 79 | 80 | // WithImage takes the image url as input and returns a Docker runner. 81 | // The image url format is the same one used in docker pull . 82 | func (d *DockerRunner) WithImage(image string) *DockerRunner { 83 | d.image = image 84 | return d 85 | } 86 | 87 | // WithSrc takes the src location and returns a Docker runner. 88 | // The src is the path to the folder that will be copied into the docker container for running the job. 89 | func (d *DockerRunner) WithSrc(src string) *DockerRunner { 90 | d.src = filepath.Clean(src) 91 | return d 92 | } 93 | 94 | // WithEnv is used to specify job variables. 95 | // Env is an array of map[string]any. The length of the map should be 1. 96 | func (d *DockerRunner) WithEnv(env []models.Variable) *DockerRunner { 97 | variables := make([]string, 0) 98 | for _, v := range env { 99 | if len(v) > 1 { 100 | log.Fatal("variables should be defined as a key value pair") 101 | } 102 | for k, v := range v { 103 | variables = append(variables, fmt.Sprintf("%s=%s", k, fmt.Sprint(v))) 104 | } 105 | } 106 | d.env = variables 107 | return d 108 | } 109 | 110 | // WithCmd specifies the script that should be run inside the container. 111 | func (d *DockerRunner) WithCmd(cmd []string) *DockerRunner { 112 | d.cmd = cmd 113 | return d 114 | } 115 | 116 | // WithEntrypoint can be used to override the default entrypoint in a docker image. 117 | func (d *DockerRunner) WithEntrypoint(entrypoint []string) *DockerRunner { 118 | d.entrypoint = entrypoint 119 | return d 120 | } 121 | 122 | // WithCredentials can be used to specify the credentials for a private image registry. 123 | func (d *DockerRunner) WithCredentials(username, password string) *DockerRunner { 124 | authConfig := registry.AuthConfig{ 125 | Username: username, 126 | Password: password, 127 | } 128 | 129 | jsonVal, err := json.Marshal(authConfig) 130 | if err != nil { 131 | log.Fatal("could not create auth config for docker authentication: ", err) 132 | } 133 | d.authConfig = base64.URLEncoding.EncodeToString(jsonVal) 134 | return d 135 | } 136 | 137 | // CreatesArtifacts is used to specify the files that will be stored as artifacts. 138 | // The input is a list of file paths wrt the src specified in WithSrc. 139 | func (d *DockerRunner) CreatesArtifacts(artifacts []string) *DockerRunner { 140 | d.artifacts = artifacts 141 | return d 142 | } 143 | 144 | // Run creates the container based on the provided configuration. 145 | func (d *DockerRunner) Run(ctx context.Context) error { 146 | cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) 147 | if err != nil { 148 | return fmt.Errorf("unable to create docker client to create container %s: %v", d.name, err) 149 | } 150 | defer cli.Close() 151 | 152 | if err := d.pullImage(ctx, cli); err != nil { 153 | return fmt.Errorf("could not pull image for container %s: %v", d.name, err) 154 | } 155 | 156 | resp, err := d.createContainer(ctx, cli) 157 | d.containerID = resp.ID 158 | if err != nil { 159 | return fmt.Errorf("unable to create container %s: %v", d.name, err) 160 | } 161 | defer func() { 162 | if rErr := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{}); rErr != nil { 163 | err = rErr 164 | } 165 | }() 166 | 167 | if err := d.createSrcDirectories(ctx, cli); err != nil { 168 | return fmt.Errorf("unable to create source directories for %s: %v", d.name, err) 169 | } 170 | 171 | if err := d.artifactManager.RetrieveArtifact(d.containerID, nil); err != nil { 172 | return fmt.Errorf("unable to retrieve artifacts for %s: %v", d.name, err) 173 | } 174 | 175 | if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { 176 | return fmt.Errorf("unable to start container %s: %v", d.name, err) 177 | } 178 | 179 | logs, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ 180 | ShowStdout: true, 181 | ShowStderr: true, 182 | Follow: true, 183 | }) 184 | if err != nil { 185 | return fmt.Errorf("unable to attach logs for %s: %v", d.name, err) 186 | } 187 | defer logs.Close() 188 | 189 | if _, err := stdcopy.StdCopy(d.dockerOptions.Stdout, d.dockerOptions.Stderr, logs); err != nil { 190 | return fmt.Errorf("unable to read container logs from %s: %v", d.name, err) 191 | } 192 | 193 | statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) 194 | select { 195 | case err := <-errCh: 196 | return fmt.Errorf("error waiting for container %s to stop: %v", d.name, err) 197 | case status := <-statusCh: 198 | if status.StatusCode != 0 { 199 | return fmt.Errorf("container %s exited with status code %d", d.name, status.StatusCode) 200 | } 201 | if err := d.publishArtifacts(); err != nil { 202 | return fmt.Errorf("unable to publish artifacts for %s: %v", d.name, err) 203 | } 204 | case <-ctx.Done(): 205 | return fmt.Errorf("context timed out, stopping container %s", d.name) 206 | } 207 | 208 | return nil 209 | } 210 | 211 | func (d *DockerRunner) createSrcDirectories(ctx context.Context, cli *client.Client) error { 212 | f, err := os.CreateTemp("", "tarcopy-*.tar") 213 | if err != nil { 214 | return err 215 | } 216 | if err := f.Close(); err != nil { 217 | return err 218 | } 219 | defer os.Remove(f.Name()) 220 | 221 | if err := utils.CompressTar(d.src, f.Name()); err != nil { 222 | return err 223 | } 224 | 225 | tar, err := os.Open(f.Name()) 226 | if err != nil { 227 | return nil 228 | } 229 | 230 | return cli.CopyToContainer(ctx, d.containerID, WORKING_DIR, tar, types.CopyToContainerOptions{}) 231 | } 232 | 233 | func (d *DockerRunner) publishArtifacts() error { 234 | for _, v := range d.artifacts { 235 | if _, err := d.artifactManager.PublishArtifact(d.containerID, filepath.Join(WORKING_DIR, v)); err != nil { 236 | return err 237 | } 238 | } 239 | return nil 240 | } 241 | 242 | func (d *DockerRunner) prepareMounts() []mount.Mount { 243 | var mounts []mount.Mount 244 | if d.dockerOptions.MountDockerSocket { 245 | mounts = append(mounts, mount.Mount{ 246 | Type: mount.TypeBind, 247 | Source: "/var/run/docker.sock", 248 | Target: "/var/run/docker.sock", 249 | }) 250 | } 251 | return mounts 252 | } 253 | 254 | func (d *DockerRunner) pullImage(ctx context.Context, cli *client.Client) error { 255 | reader, err := cli.ImagePull(ctx, d.image, types.ImagePullOptions{RegistryAuth: d.authConfig}) 256 | if err != nil { 257 | return err 258 | } 259 | defer reader.Close() 260 | 261 | imageLogs := io.Discard 262 | if d.dockerOptions.ShowImagePull { 263 | imageLogs = d.dockerOptions.Stdout 264 | } 265 | if _, err := io.Copy(imageLogs, reader); err != nil { 266 | return err 267 | } 268 | 269 | return nil 270 | } 271 | 272 | func (d *DockerRunner) createContainer(ctx context.Context, cli *client.Client) (container.CreateResponse, error) { 273 | commandScript := strings.Join(d.cmd, "\n") 274 | cmd := []string{"/bin/sh", "-c", commandScript} 275 | if len(d.entrypoint) > 0 { 276 | cmd = []string{commandScript} 277 | } 278 | 279 | resp, err := cli.ContainerCreate(ctx, &container.Config{ 280 | Image: d.image, 281 | Env: d.env, 282 | Entrypoint: d.entrypoint, 283 | Cmd: cmd, 284 | WorkingDir: WORKING_DIR, 285 | }, &container.HostConfig{ 286 | Mounts: d.prepareMounts(), 287 | }, nil, nil, d.name) 288 | if err != nil { 289 | return container.CreateResponse{}, err 290 | } 291 | return resp, nil 292 | } 293 | -------------------------------------------------------------------------------- /pkg/runner/docker_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | "regexp" 11 | "strings" 12 | "testing" 13 | "time" 14 | 15 | "github.com/opnlabs/dot/pkg/artifacts" 16 | "github.com/opnlabs/dot/pkg/models" 17 | "github.com/opnlabs/dot/pkg/utils" 18 | "github.com/stretchr/testify/assert" 19 | ) 20 | 21 | type Test struct { 22 | Name string 23 | Manager artifacts.ArtifactManager 24 | Image string 25 | Src string 26 | Entrypoint []string 27 | Script []string 28 | Variables []models.Variable 29 | Artifacts []string 30 | Output io.Writer 31 | Ctx context.Context 32 | Expectation func(*testing.T, *bytes.Buffer) bool 33 | Username string 34 | Password string 35 | } 36 | 37 | func teardown(tb testing.TB) { 38 | 39 | wd, err := os.Getwd() 40 | if err != nil { 41 | log.Println(err) 42 | return 43 | } 44 | os.RemoveAll(filepath.Join(wd, ".artifacts")) 45 | } 46 | 47 | func TestRun(t *testing.T) { 48 | 49 | var b bytes.Buffer 50 | ctx := context.Background() 51 | manager := artifacts.NewDockerArtifactsManager(".artifacts") 52 | 53 | // ctxTimeout, cancel := context.WithTimeout(ctx, time.Millisecond) 54 | // defer cancel() 55 | 56 | tests := []Test{ 57 | { 58 | Name: "Test Image", 59 | Manager: manager, 60 | Image: "docker.io/alpine", 61 | Script: []string{ 62 | "cat /etc/os-release", 63 | }, 64 | Output: &b, 65 | Expectation: testImageOutput, 66 | Ctx: ctx, 67 | Username: "", 68 | Password: "", 69 | }, 70 | { 71 | Name: "Test Variables", 72 | Manager: manager, 73 | Image: "docker.io/alpine", 74 | Variables: []models.Variable{ 75 | map[string]any{ 76 | "TESTING_VARIABLE": "TESTING", 77 | }, 78 | }, 79 | Script: []string{ 80 | "echo $TESTING_VARIABLE", 81 | }, 82 | Output: &b, 83 | Expectation: testVariableOutput, 84 | Ctx: ctx, 85 | }, 86 | { 87 | Name: "Test Create Artifact", 88 | Manager: manager, 89 | Image: "docker.io/alpine", 90 | Script: []string{ 91 | "echo TESTING >> log.txt", 92 | }, 93 | Output: &b, 94 | Artifacts: []string{ 95 | "log.txt", 96 | }, 97 | Expectation: testArtifactCreation, 98 | Ctx: ctx, 99 | }, 100 | { 101 | Name: "Test Retrieve Artifact", 102 | Manager: manager, 103 | Image: "docker.io/alpine", 104 | Script: []string{ 105 | "cat log.txt", 106 | }, 107 | Output: &b, 108 | Expectation: testVariableOutput, 109 | Ctx: ctx, 110 | }, 111 | { 112 | Name: "Test Entrypoint", 113 | Manager: manager, 114 | Image: "docker.io/alpine", 115 | Entrypoint: []string{"echo"}, 116 | Script: []string{ 117 | "TESTING", 118 | }, 119 | Output: &b, 120 | Expectation: testVariableOutput, 121 | Ctx: ctx, 122 | }, 123 | } 124 | 125 | for _, test := range tests { 126 | b.Truncate(0) 127 | err := NewDockerRunner(test.Name, test.Manager, DockerRunnerOptions{ShowImagePull: false, Stdout: test.Output, Stderr: os.Stderr}). 128 | WithImage(test.Image). 129 | WithSrc(test.Src). 130 | WithEntrypoint(test.Entrypoint). 131 | WithCmd(test.Script). 132 | WithEnv(test.Variables). 133 | WithCredentials(test.Username, test.Password). 134 | CreatesArtifacts(test.Artifacts).Run(test.Ctx) 135 | assert.NoError(t, err, "error is nil") 136 | assert.Equal(t, true, test.Expectation(t, &b)) 137 | } 138 | 139 | teardown(t) 140 | } 141 | 142 | func TestMountDockerSocket(t *testing.T) { 143 | manager := artifacts.NewDockerArtifactsManager(".artifacts") 144 | err := NewDockerRunner("Mount Docker Socket Test", manager, DockerRunnerOptions{MountDockerSocket: true, ShowImagePull: false, Stdout: nil, Stderr: nil}). 145 | WithImage("docker.io/alpine"). 146 | Run(context.Background()) 147 | if err != nil { 148 | t.Error(err) 149 | } 150 | teardown(t) 151 | } 152 | 153 | func TestNonExistingSrcDirectory(t *testing.T) { 154 | manager := artifacts.NewDockerArtifactsManager(".artifacts") 155 | err := NewDockerRunner("Non existing src directory", manager, DockerRunnerOptions{ShowImagePull: false, Stdout: nil, Stderr: nil}). 156 | WithImage("docker.io/alpine"). 157 | WithSrc("testnonexisting"). 158 | Run(context.Background()) 159 | assert.ErrorContains(t, err, "unable to create source directories") 160 | teardown(t) 161 | } 162 | 163 | func TestPublishNonExistingFile(t *testing.T) { 164 | manager := artifacts.NewDockerArtifactsManager(".artifacts") 165 | err := NewDockerRunner("Non existing artifact publish", manager, DockerRunnerOptions{ShowImagePull: false, Stdout: nil, Stderr: nil}). 166 | WithImage("docker.io/alpine"). 167 | CreatesArtifacts([]string{"testing123"}). 168 | Run(context.Background()) 169 | assert.ErrorContains(t, err, "unable to publish artifacts") 170 | teardown(t) 171 | } 172 | 173 | func TestTimeout(t *testing.T) { 174 | manager := artifacts.NewDockerArtifactsManager(".artifacts") 175 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) 176 | defer cancel() 177 | 178 | err := NewDockerRunner("Test Timeout", manager, DockerRunnerOptions{ShowImagePull: false, Stdout: nil, Stderr: nil}). 179 | WithImage("docker.io/alpine"). 180 | WithCmd([]string{"sleep", "60"}). 181 | Run(ctx) 182 | assert.ErrorContains(t, err, "container test-timeout") 183 | teardown(t) 184 | } 185 | 186 | func testImageOutput(t *testing.T, b *bytes.Buffer) bool { 187 | str := b.String() 188 | lines := strings.Split(str, "\n") 189 | 190 | if len(lines) < 1 { 191 | t.Error("output lines less than expected") 192 | return false 193 | } 194 | name := strings.Split(lines[0], "=") 195 | if len(name) != 2 { 196 | t.Error("name field not found") 197 | return false 198 | } 199 | 200 | return (strings.Compare(strings.Trim(name[1], "\""), "Alpine Linux") == 0) 201 | 202 | } 203 | 204 | func testVariableOutput(t *testing.T, b *bytes.Buffer) bool { 205 | str := b.String() 206 | str = regexp.MustCompile(`[^a-zA-Z0-9 ]+`).ReplaceAllString(str, "") 207 | return (strings.Compare(strings.TrimSpace(str), "TESTING") == 0) 208 | } 209 | 210 | func testArtifactCreation(t *testing.T, b *bytes.Buffer) bool { 211 | wd, err := os.Getwd() 212 | if err != nil { 213 | t.Error(err) 214 | return false 215 | } 216 | 217 | files, err := os.ReadDir(filepath.Join(wd, ".artifacts")) 218 | if err != nil { 219 | t.Error(err) 220 | return false 221 | } 222 | for _, f := range files { 223 | err := utils.DecompressTar(filepath.Join(wd, ".artifacts", f.Name()), filepath.Join(wd, ".artifacts")) 224 | if err != nil { 225 | t.Error(err) 226 | } 227 | 228 | logFile, err := os.ReadFile(filepath.Join(wd, ".artifacts", "log.txt")) 229 | if err != nil { 230 | t.Error(err) 231 | } 232 | testing := regexp.MustCompile(`[^a-zA-Z0-9 ]+`).ReplaceAllString(string(logFile), "") 233 | if strings.Compare(strings.TrimSpace(testing), "TESTING") == 0 { 234 | return true 235 | } 236 | } 237 | return false 238 | } 239 | 240 | // func testTimeoutOutput(t *testing.T, b *bytes.Buffer) bool { 241 | // str := b.String() 242 | // str = regexp.MustCompile(`[^a-zA-Z0-9 ]+`).ReplaceAllString(str, "") 243 | // return (strings.Compare(strings.TrimSpace(str), "context timed out") == 0) 244 | // } 245 | -------------------------------------------------------------------------------- /pkg/store/memorystore.go: -------------------------------------------------------------------------------- 1 | // Package store implements a simple key-value store. 2 | package store 3 | 4 | import ( 5 | "errors" 6 | "log" 7 | "sync" 8 | ) 9 | 10 | var ( 11 | ErrKeyExists = errors.New("store: key already exists") 12 | ErrKeyDoesntExist = errors.New("store: key does not exist") 13 | ) 14 | 15 | type Store interface { 16 | Set(key string, value interface{}) error 17 | Get(key string) (interface{}, error) 18 | Delete(key string) error 19 | Update(key string, newValue interface{}) error 20 | } 21 | 22 | type MemStore struct { 23 | lock *sync.Mutex 24 | store map[string]interface{} 25 | } 26 | 27 | var memStore *MemStore 28 | 29 | func NewMemStore() Store { 30 | if memStore != nil { 31 | return memStore 32 | } 33 | 34 | memStore = &MemStore{ 35 | lock: new(sync.Mutex), 36 | store: make(map[string]interface{}), 37 | } 38 | 39 | return memStore 40 | } 41 | 42 | // Set is used to set a value to a key. 43 | func (m *MemStore) Set(key string, value interface{}) error { 44 | m.lock.Lock() 45 | defer m.lock.Unlock() 46 | 47 | if _, ok := m.store[key]; ok { 48 | return ErrKeyExists 49 | } 50 | m.store[key] = value 51 | return nil 52 | } 53 | 54 | // Get is used to get a value from a key. 55 | func (m *MemStore) Get(key string) (interface{}, error) { 56 | m.lock.Lock() 57 | defer m.lock.Unlock() 58 | log.Println(m.store[key]) 59 | 60 | if _, ok := m.store[key]; !ok { 61 | return nil, ErrKeyDoesntExist 62 | } 63 | return m.store[key], nil 64 | } 65 | 66 | // Delete removes the specified key and value. 67 | func (m *MemStore) Delete(key string) error { 68 | m.lock.Lock() 69 | defer m.lock.Unlock() 70 | 71 | if _, ok := m.store[key]; !ok { 72 | return ErrKeyDoesntExist 73 | } 74 | delete(m.store, key) 75 | return nil 76 | } 77 | 78 | // Update can be used to change the value for a given key. 79 | func (m *MemStore) Update(key string, value interface{}) error { 80 | m.lock.Lock() 81 | defer m.lock.Unlock() 82 | 83 | if _, ok := m.store[key]; !ok { 84 | return ErrKeyDoesntExist 85 | } 86 | m.store[key] = value 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /pkg/store/memorystore_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | const ( 8 | KEY1 = "test-key" 9 | KEY2 = "test-key2" 10 | VALUE1 = "TESTING123" 11 | VALUE2 = "TESTING234" 12 | NEWVALUE = "NEWVALUE" 13 | NONEXISTINGKEY = "12345" 14 | ) 15 | 16 | func TestSet(t *testing.T) { 17 | memStore := NewMemStore() 18 | 19 | err := memStore.Set(KEY1, VALUE1) 20 | if err != nil { 21 | t.Error(err, "could not set key") 22 | } 23 | 24 | err = memStore.Set(KEY1, VALUE2) 25 | if err != ErrKeyExists { 26 | t.Error("did not return the key exists error") 27 | } 28 | } 29 | 30 | func TestGet(t *testing.T) { 31 | memStore := NewMemStore() 32 | 33 | err := memStore.Set(KEY2, VALUE2) 34 | if err != nil { 35 | t.Error(err, "could not set key") 36 | } 37 | 38 | val, err := memStore.Get(KEY2) 39 | if err != nil { 40 | t.Error(err) 41 | } 42 | if val.(string) != VALUE2 { 43 | t.Errorf("retrieved value not the same, expected %s got %s", VALUE2, val.(string)) 44 | } 45 | } 46 | 47 | func TestGetNonExistingKey(t *testing.T) { 48 | memStore := NewMemStore() 49 | 50 | _, err := memStore.Get(NONEXISTINGKEY) 51 | if err != ErrKeyDoesntExist { 52 | t.Error("did not return key doesn't exist error") 53 | } 54 | } 55 | 56 | func TestPreviousEntries(t *testing.T) { 57 | memStore := NewMemStore() 58 | 59 | val, err := memStore.Get(KEY1) 60 | if err != nil { 61 | t.Error(err) 62 | } 63 | if val.(string) != VALUE1 { 64 | t.Errorf("expected %s, got %s", VALUE1, val.(string)) 65 | } 66 | } 67 | 68 | func TestDelete(t *testing.T) { 69 | memStore := NewMemStore() 70 | 71 | err := memStore.Delete(KEY2) 72 | if err != nil { 73 | t.Error(err) 74 | } 75 | _, err = memStore.Get(KEY2) 76 | if err != ErrKeyDoesntExist { 77 | t.Error("delete did not remove the key") 78 | } 79 | } 80 | 81 | func TestUpdate(t *testing.T) { 82 | memStore := NewMemStore() 83 | err := memStore.Update(KEY1, NEWVALUE) 84 | if err != nil { 85 | t.Error(err) 86 | } 87 | val, err := memStore.Get("test-key") 88 | if err != nil { 89 | t.Error(err) 90 | } 91 | if val.(string) != NEWVALUE { 92 | t.Errorf("expected %s, got %s", NEWVALUE, val.(string)) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pkg/utils/compress.go: -------------------------------------------------------------------------------- 1 | // Package utils provides some utility functions to compress and decompress tar and tar.gz. 2 | // It also provides a logger that can output in color and implements io.Writer. 3 | package utils 4 | 5 | import ( 6 | "archive/tar" 7 | "compress/gzip" 8 | "fmt" 9 | "io" 10 | "io/fs" 11 | "os" 12 | "path/filepath" 13 | "strings" 14 | ) 15 | 16 | const MaxFileSizeBytes = 50 * 1024 * 1024 // 50MB 17 | 18 | // Compress takes a path to a file or directory and creates a .tar.gzip file at the outputPath location. 19 | func Compress(path, outputPath string) error { 20 | tarFile, err := os.Create(filepath.Clean(outputPath)) 21 | if err != nil { 22 | return fmt.Errorf("could not create tar.gzip file %s: %v", outputPath, err) 23 | } 24 | defer tarFile.Close() 25 | 26 | gzw := gzip.NewWriter(tarFile) 27 | defer gzw.Close() 28 | 29 | tw := tar.NewWriter(gzw) 30 | defer tw.Close() 31 | 32 | return filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { 33 | if err != nil { 34 | return err 35 | } 36 | 37 | header, err := tar.FileInfoHeader(info, path) 38 | if err != nil { 39 | return fmt.Errorf("could not create tar.gzip file %s: %v", path, err) 40 | } 41 | header.Name = filepath.ToSlash(path) 42 | if err := tw.WriteHeader(header); err != nil { 43 | return fmt.Errorf("could not create tar.gzip file %s: %v", path, err) 44 | } 45 | 46 | if !info.IsDir() { 47 | data, err := os.Open(path) 48 | if err != nil { 49 | return fmt.Errorf("could not open file %s: %v", path, err) 50 | } 51 | if _, err := io.Copy(tw, data); err != nil { 52 | return fmt.Errorf("could not copy tar.gzip contents for file %s: %v", path, err) 53 | } 54 | if err := data.Close(); err != nil { 55 | return fmt.Errorf("could not close file %s: %v", data.Name(), err) 56 | } 57 | } 58 | return nil 59 | }) 60 | } 61 | 62 | // Decompress takes a location to a .tar.gzip file and a base path and decompresses the contents wrt the base path. 63 | func Decompress(tarPath, baseDir string) error { 64 | tarFile, err := os.Open(filepath.Clean(tarPath)) 65 | if err != nil { 66 | return fmt.Errorf("could not open tar.gzip file %s: %v", tarPath, err) 67 | } 68 | defer tarFile.Close() 69 | 70 | gzr, err := gzip.NewReader(tarFile) 71 | if err != nil { 72 | return fmt.Errorf("could not read tar.gzip file %s: %v", tarPath, err) 73 | } 74 | defer gzr.Close() 75 | 76 | tr := tar.NewReader(gzr) 77 | for { 78 | header, err := tr.Next() 79 | if err == io.EOF { 80 | return nil 81 | } else if err != nil { 82 | return fmt.Errorf("could not read tar.gzip header %s: %v", header.Name, err) 83 | } 84 | 85 | target, err := sanitizeArchivePath(baseDir, header.Name) 86 | if err != nil { 87 | return err 88 | } 89 | switch header.Typeflag { 90 | case tar.TypeDir: 91 | if _, err := os.Stat(target); err != nil { 92 | if err := os.MkdirAll(target, fs.FileMode(header.Mode)); err != nil { 93 | return fmt.Errorf("could not create dir %s: %v", target, err) 94 | } 95 | } 96 | case tar.TypeReg: 97 | f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, 0644) 98 | if err != nil { 99 | return fmt.Errorf("could not open file %s: %v", target, err) 100 | } 101 | defer f.Close() 102 | 103 | if _, err := io.CopyN(f, tr, MaxFileSizeBytes); err != nil && err != io.EOF { 104 | return fmt.Errorf("could not copy tar.gzip contents to file %s: %v", target, err) 105 | } 106 | } 107 | } 108 | } 109 | 110 | // CompressTar takes a path to a file or directory and creates a .tar file at the outputPath location. 111 | func CompressTar(path, outputPath string) error { 112 | tarFile, err := os.Create(filepath.Clean(outputPath)) 113 | if err != nil { 114 | return fmt.Errorf("could not create tar.gzip file %s: %v", outputPath, err) 115 | } 116 | defer tarFile.Close() 117 | 118 | tw := tar.NewWriter(tarFile) 119 | defer tw.Close() 120 | 121 | return filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { 122 | if err != nil { 123 | return err 124 | } 125 | 126 | header, err := tar.FileInfoHeader(info, path) 127 | if err != nil { 128 | return fmt.Errorf("could not create tar.gzip file %s: %v", path, err) 129 | } 130 | header.Name = filepath.ToSlash(path) 131 | if err := tw.WriteHeader(header); err != nil { 132 | return fmt.Errorf("could not create tar.gzip file %s: %v", path, err) 133 | } 134 | 135 | if !info.IsDir() { 136 | data, err := os.Open(path) 137 | if err != nil { 138 | return fmt.Errorf("could not open file %s: %v", path, err) 139 | } 140 | if _, err := io.Copy(tw, data); err != nil { 141 | return fmt.Errorf("could not copy tar.gzip contents for file %s: %v", path, err) 142 | } 143 | if err := data.Close(); err != nil { 144 | return fmt.Errorf("could not close file %s: %v", data.Name(), err) 145 | } 146 | } 147 | return nil 148 | }) 149 | } 150 | 151 | // DecompressTar takes a location to a .tar file and a base path and decompresses the contents wrt the base path. 152 | func DecompressTar(tarPath, baseDir string) error { 153 | tarFile, err := os.Open(filepath.Clean(tarPath)) 154 | if err != nil { 155 | return fmt.Errorf("could not open tar file %s: %v", tarPath, err) 156 | } 157 | defer tarFile.Close() 158 | 159 | tr := tar.NewReader(tarFile) 160 | for { 161 | header, err := tr.Next() 162 | if err == io.EOF { 163 | return nil 164 | } else if err != nil { 165 | return fmt.Errorf("could not read tar header %s: %v", header.Name, err) 166 | } 167 | 168 | target, err := sanitizeArchivePath(baseDir, header.Name) 169 | if err != nil { 170 | return err 171 | } 172 | switch header.Typeflag { 173 | case tar.TypeDir: 174 | if _, err := os.Stat(target); err != nil { 175 | if err := os.MkdirAll(target, fs.FileMode(header.Mode)); err != nil { 176 | return fmt.Errorf("could not create dir %s: %v", target, err) 177 | } 178 | } 179 | case tar.TypeReg: 180 | f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, 0644) 181 | if err != nil { 182 | return fmt.Errorf("could not open file %s: %v", target, err) 183 | } 184 | defer f.Close() 185 | 186 | if _, err := io.CopyN(f, tr, MaxFileSizeBytes); err != nil && err != io.EOF { 187 | return fmt.Errorf("could not copy tar contents to file %s: %v", target, err) 188 | } 189 | } 190 | } 191 | } 192 | 193 | // TarCopy uses tar archive to copy src to dst to preserve the folder structure. 194 | func TarCopy(src, dst, tempDir string) error { 195 | f, err := os.CreateTemp(tempDir, "tarcopy-*.tar.gzip") 196 | if err != nil { 197 | return fmt.Errorf("could not create tar.gzip file in %s: %v", tempDir, err) 198 | } 199 | if err := f.Close(); err != nil { 200 | return fmt.Errorf("could not close file %s: %v", f.Name(), err) 201 | } 202 | 203 | if err := Compress(src, f.Name()); err != nil { 204 | return fmt.Errorf("could not create %s from src %s: %v", f.Name(), src, err) 205 | } 206 | 207 | if err := Decompress(f.Name(), dst); err != nil { 208 | return fmt.Errorf("could not decompress %s to dst %s: %v", f.Name(), dst, err) 209 | } 210 | 211 | return nil 212 | } 213 | 214 | func sanitizeArchivePath(dir, target string) (string, error) { 215 | full := filepath.Join(dir, target) 216 | if strings.HasPrefix(full, filepath.Clean(dir)) { 217 | return full, nil 218 | } 219 | 220 | return "", fmt.Errorf("illegal path %s", full) 221 | } 222 | -------------------------------------------------------------------------------- /pkg/utils/logger.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | 7 | "github.com/fatih/color" 8 | ) 9 | 10 | var colors = []color.Attribute{color.FgYellow, color.FgGreen, color.FgRed, color.FgWhite, color.FgMagenta} 11 | var index = -1 12 | 13 | var l sync.Mutex 14 | 15 | const MaxNameLength = 20 16 | 17 | // ColorLogger provides an io.Writer that can output in color. 18 | type ColorLogger struct { 19 | name string 20 | writer io.Writer 21 | c color.Attribute 22 | } 23 | 24 | func NewColorLogger(name string, writer io.Writer, newColor bool) io.Writer { 25 | if newColor { 26 | l.Lock() 27 | defer l.Unlock() 28 | index = (index + 1) % len(colors) 29 | } 30 | 31 | if len(name) > MaxNameLength { 32 | name = name[:MaxNameLength-3] + "..." 33 | } 34 | 35 | return &ColorLogger{ 36 | name: name, 37 | writer: writer, 38 | c: colors[index], 39 | } 40 | } 41 | 42 | func (c *ColorLogger) Write(p []byte) (int, error) { 43 | out := color.New(c.c) 44 | out.Print(c.name, " | ") 45 | return out.Fprintf(c.writer, "%s", p) 46 | } 47 | --------------------------------------------------------------------------------