├── .github └── workflows │ ├── build.yml │ ├── coverage.yml │ └── docker-publish.yaml ├── .gitignore ├── .goreleaser.yaml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── README.md ├── cmd └── togomak │ ├── main.go │ └── utils.go ├── docs ├── README.md └── img │ ├── screenshot-cover.png │ └── screenshot.png ├── examples ├── README.md ├── ansi │ ├── .meta.yaml │ └── togomak.hcl ├── colors │ └── togomak.hcl ├── conditions │ ├── .meta.yaml │ └── togomak.hcl ├── demo │ ├── .meta.yaml │ ├── screenrecord.gif │ └── togomak.hcl ├── docker-entrypoint │ ├── .meta.yaml │ └── togomak.hcl ├── docker │ ├── .meta.yaml │ ├── diary │ │ └── shinji.diary.txt │ ├── rei.diary.txt │ └── togomak.hcl ├── env │ ├── .meta.yaml │ └── togomak.hcl ├── files │ ├── .meta.yaml │ └── togomak.hcl ├── for-each-map │ ├── .meta.yaml │ └── togomak.hcl ├── for-each-module │ ├── .meta.yaml │ └── togomak.hcl ├── functions │ ├── .meta.yaml │ └── togomak.hcl ├── git │ ├── .meta.yaml │ └── togomak.hcl ├── git_tags │ ├── .meta.yaml │ └── togomak.hcl ├── hooks │ ├── .meta.yaml │ └── togomak.hcl ├── import │ ├── .meta.yaml │ ├── module │ │ ├── child │ │ │ └── togomak.hcl │ │ └── togomak.hcl │ └── togomak.hcl ├── lifecycles │ ├── .meta.yaml │ └── togomak.hcl ├── locals │ ├── .meta.yaml │ └── togomak.hcl ├── macros │ ├── .meta.yaml │ └── togomak.hcl ├── module-local │ ├── .meta.yaml │ ├── calculator │ │ └── togomak.hcl │ └── togomak.hcl ├── module-phases-inheritance │ ├── calculator │ │ └── togomak.hcl │ └── togomak.hcl ├── modules │ ├── .meta.yaml │ ├── module │ │ └── togomak.hcl │ └── togomak.hcl ├── multiple-files │ ├── stage1.hcl │ ├── stage2.hcl │ └── togomak.hcl ├── nested-filter │ ├── child │ │ └── togomak.hcl │ └── togomak.hcl ├── output │ └── togomak.hcl ├── pre-post │ ├── .meta.yaml │ └── togomak.hcl ├── prompt │ ├── .meta.yaml │ └── togomak.hcl ├── recursive │ ├── .meta.yaml │ ├── bye │ │ └── togomak.hcl │ ├── hello │ │ └── togomak.hcl │ └── togomak.hcl ├── remote-stages │ ├── .gitignore │ └── togomak.hcl ├── terraform │ ├── .meta.yaml │ ├── .terraform.lock.hcl │ ├── random.tf │ └── togomak.hcl ├── togomak.hcl └── variables │ ├── calculator │ └── togomak.hcl │ └── togomak.hcl ├── go.mod ├── go.sum ├── internal ├── behavior │ └── models.go ├── blocks │ ├── data │ │ ├── env.go │ │ ├── file.go │ │ ├── git.go │ │ ├── prompt.go │ │ ├── provider.go │ │ ├── provider_options.go │ │ ├── provider_stdlib.go │ │ └── terraform.go │ └── types.go ├── c │ ├── context.go │ └── context_test.go ├── cache │ └── clean.go ├── ci │ ├── builder.go │ ├── conductor.go │ ├── conductor_config.go │ ├── conductor_context.go │ ├── conductor_hcl.go │ ├── context.go │ ├── daemon.go │ ├── data.go │ ├── data_hcl.go │ ├── data_lifecycle.go │ ├── data_logging.go │ ├── data_prop.go │ ├── data_prop_test.go │ ├── data_provider.go │ ├── data_retry.go │ ├── data_retry_test.go │ ├── data_run.go │ ├── data_schema.go │ ├── data_test.go │ ├── graph.go │ ├── handler.go │ ├── hooks.go │ ├── import.go │ ├── import_expand.go │ ├── import_schema.go │ ├── lifecycle.go │ ├── local_logging.go │ ├── locals.go │ ├── locals_hcl.go │ ├── locals_lifecycle.go │ ├── locals_prop.go │ ├── locals_prop_test.go │ ├── locals_retry.go │ ├── locals_retry_test.go │ ├── locals_run.go │ ├── locals_schema.go │ ├── locals_test.go │ ├── macro.go │ ├── macro_lifecycle.go │ ├── macro_logging.go │ ├── macro_prop.go │ ├── macro_prop_test.go │ ├── macro_retry.go │ ├── macro_retry_test.go │ ├── macro_run.go │ ├── macro_schema.go │ ├── module.go │ ├── module_hcl.go │ ├── module_lifecycle.go │ ├── module_logger.go │ ├── module_prop.go │ ├── module_retry.go │ ├── module_run.go │ ├── module_schema.go │ ├── output.go │ ├── pipeline.go │ ├── pipeline_expand.go │ ├── pipeline_import.go │ ├── pipeline_meta.go │ ├── pipeline_outputs.go │ ├── pipeline_read.go │ ├── pipeline_run.go │ ├── prop.go │ ├── query_model.go │ ├── query_slice.go │ ├── run.go │ ├── run_test.go │ ├── runnable.go │ ├── stage.go │ ├── stage_containers_hcl.go │ ├── stage_hcl.go │ ├── stage_hooks.go │ ├── stage_lifecycle.go │ ├── stage_logging.go │ ├── stage_prop.go │ ├── stage_prop_test.go │ ├── stage_retry.go │ ├── stage_run.go │ ├── stage_schema.go │ ├── stage_terminate.go │ ├── stage_test.go │ ├── variable.go │ ├── variable_hcl.go │ ├── variable_parse.go │ ├── variable_retry.go │ ├── variable_run.go │ └── variable_schema.go ├── conductor │ └── impl.go ├── core │ └── togomak.go ├── dg │ ├── wrapper.go │ └── wrapper_test.go ├── filter │ ├── filter.go │ ├── filter_model.go │ ├── filter_unmarshal_test.go │ └── operations.go ├── global │ ├── global.go │ ├── hcl.go │ ├── logging.go │ └── mutex.go ├── log │ └── logger.go ├── logging │ ├── google_cloud.go │ └── logging.go ├── meta │ └── app.go ├── orchestra │ ├── context.go │ ├── format.go │ ├── imports.go │ ├── init.go │ ├── list.go │ ├── logging.go │ ├── orchestra.go │ └── utils.go ├── parse │ ├── eval_context.go │ └── parse.go ├── path │ └── models.go ├── rules │ ├── model.go │ └── operation_test.go ├── runnable │ ├── options.go │ └── status.go ├── third-party │ ├── docker │ │ └── docker.go │ └── hashicorp │ │ └── terraform │ │ ├── README.md │ │ └── lang │ │ ├── funcs │ │ ├── collection.go │ │ ├── collection_test.go │ │ ├── conversion.go │ │ ├── conversion_test.go │ │ ├── crypto.go │ │ ├── crypto_test.go │ │ ├── datetime.go │ │ ├── datetime_test.go │ │ ├── descriptions.go │ │ ├── encoding.go │ │ ├── encoding_test.go │ │ ├── filesystem.go │ │ ├── filesystem_test.go │ │ ├── number.go │ │ ├── number_test.go │ │ ├── redact.go │ │ ├── redact_test.go │ │ ├── refinements.go │ │ ├── sensitive.go │ │ ├── sensitive_test.go │ │ ├── string.go │ │ ├── string_test.go │ │ └── testdata │ │ │ ├── bare.tmpl │ │ │ ├── func.tmpl │ │ │ ├── hello.tmpl │ │ │ ├── hello.txt │ │ │ ├── icon.png │ │ │ ├── list.tmpl │ │ │ ├── recursive.tmpl │ │ │ └── unreadable │ │ │ └── foobar │ │ ├── marks │ │ └── marks.go │ │ └── types │ │ ├── type_type.go │ │ └── types.go ├── ui │ ├── colors.go │ ├── getter.go │ ├── passive.go │ ├── progress.go │ └── prompt.go └── x │ ├── blocks.go │ ├── error.go │ ├── error_test.go │ └── file.go ├── tests ├── .gitignore ├── prompt_test.go ├── tests │ └── failing │ │ ├── cant-run-fail │ │ └── togomak.hcl │ │ ├── dependency-cycles │ │ └── togomak.hcl │ │ ├── dependency-fail │ │ └── togomak.hcl │ │ ├── env-data-key-missing │ │ └── togomak.hcl │ │ ├── failing-hooks │ │ └── togomak.hcl │ │ ├── invalid-block-id │ │ └── togomak.hcl │ │ ├── invalid-env-output │ │ └── togomak.hcl │ │ ├── invalid-path-import │ │ └── togomak.hcl │ │ ├── invalid-stage-macro-invalid-file │ │ └── togomak.hcl │ │ ├── invalid-stage-multiple-macros │ │ └── togomak.hcl │ │ ├── nested-failing-tests │ │ ├── nested.hcl │ │ └── togomak.hcl │ │ ├── no-script-no-args │ │ └── togomak.hcl │ │ ├── retry │ │ └── togomak.hcl │ │ └── self-referenced-dependencies │ │ └── togomak.hcl └── togomak.hcl └── togomak.hcl /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*.*' 7 | pull_request: 8 | branches: 9 | - 'v1' 10 | - 'develop' 11 | 12 | permissions: 13 | contents: write 14 | 15 | jobs: 16 | goreleaser: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - 20 | name: Checkout 21 | uses: actions/checkout@v2 22 | with: 23 | fetch-depth: 0 24 | - 25 | name: Set up Go 26 | uses: actions/setup-go@v2 27 | - 28 | name: Run GoReleaser 29 | uses: goreleaser/goreleaser-action@v4 30 | with: 31 | # either 'goreleaser' (default) or 'goreleaser-pro' 32 | distribution: goreleaser 33 | version: latest 34 | args: release --clean --skip-publish --skip-validate 35 | env: 36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 37 | - 38 | name: Upload assets 39 | uses: actions/upload-artifact@v3 40 | with: 41 | name: togomak-artifacts 42 | path: dist/* 43 | - 44 | name: Run GoReleaser 45 | uses: goreleaser/goreleaser-action@v4 46 | if: startsWith(github.ref, 'refs/tags/') 47 | with: 48 | # either 'goreleaser' (default) or 'goreleaser-pro' 49 | distribution: goreleaser 50 | version: latest 51 | args: release --skip-validate --clean 52 | env: 53 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 54 | 55 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Test and coverage 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | with: 11 | fetch-depth: 2 12 | - uses: actions/setup-go@v2 13 | with: 14 | go-version: '1.20' 15 | 16 | - uses: hashicorp/setup-terraform@v2 17 | with: 18 | terraform_version: 1.5.5 19 | terraform_wrapper: false 20 | 21 | - name: Build 22 | run: go build -o ./togomak ./cmd/togomak/. 23 | - name: Coverage 24 | run: ./togomak -C tests 25 | - name: Upload coverage to Codecov 26 | uses: codecov/codecov-action@v3 27 | with: 28 | fail_ci_if_error: false 29 | token: ${{ secrets.CODECOV_TOKEN }} 30 | verbose: true 31 | functionalities: search 32 | files: ./coverage_unit_tests.out,./tests/coverage.out 33 | -------------------------------------------------------------------------------- /.github/workflows/docker-publish.yaml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | # This workflow uses actions that are not certified by GitHub. 4 | # They are provided by a third-party and are governed by 5 | # separate terms of service, privacy policy, and support 6 | # documentation. 7 | 8 | on: 9 | schedule: 10 | - cron: '33 12 * * *' 11 | push: 12 | branches: [ "v2", "v1", "develop" ] 13 | # Publish semver tags as releases. 14 | tags: [ 'v*.*.*' ] 15 | pull_request: 16 | branches: [ "v2", "v1", "develop" ] 17 | 18 | env: 19 | # Use docker.io for Docker Hub if empty 20 | REGISTRY: ghcr.io 21 | # github.repository as / 22 | IMAGE_NAME: ${{ github.repository }} 23 | 24 | 25 | jobs: 26 | build: 27 | 28 | runs-on: ubuntu-latest 29 | permissions: 30 | contents: read 31 | packages: write 32 | # This is used to complete the identity challenge 33 | # with sigstore/fulcio when running outside of PRs. 34 | id-token: write 35 | strategy: 36 | matrix: 37 | type: 38 | - docker 39 | - tiny 40 | - alpine 41 | - docker-buster 42 | - default 43 | 44 | steps: 45 | - name: Checkout repository 46 | uses: actions/checkout@v3 47 | 48 | # Install the cosign tool except on PR 49 | # https://github.com/sigstore/cosign-installer 50 | - name: Install Cosign 51 | uses: sigstore/cosign-installer@v3.1.1 52 | with: 53 | cosign-release: 'v2.1.1' 54 | 55 | # Workaround: https://github.com/docker/build-push-action/issues/461 56 | - name: Setup Docker buildx 57 | uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf 58 | 59 | # Login against a Docker registry except on PR 60 | # https://github.com/docker/login-action 61 | - name: Log into registry ${{ env.REGISTRY }} 62 | if: github.event_name != 'pull_request' 63 | uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c 64 | with: 65 | registry: ${{ env.REGISTRY }} 66 | username: ${{ github.actor }} 67 | password: ${{ secrets.GITHUB_TOKEN }} 68 | 69 | # Extract metadata (tags, labels) for Docker 70 | # https://github.com/docker/metadata-action 71 | - name: Extract Docker metadata 72 | id: meta 73 | uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 74 | with: 75 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 76 | tags: | 77 | type=schedule 78 | type=semver,pattern={{version}} 79 | type=semver,pattern={{major}}.{{minor}} 80 | type=semver,pattern={{major}} 81 | type=ref,event=branch 82 | type=ref,event=pr 83 | flavor: | 84 | suffix=${{ matrix.type == 'default' && '' || matrix.type }} 85 | 86 | # Build and push Docker image with Buildx (don't push on PR) 87 | # https://github.com/docker/build-push-action 88 | - name: Build and push Docker image 89 | id: build-and-push 90 | uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 91 | with: 92 | context: . 93 | push: ${{ github.event_name != 'pull_request' }} 94 | tags: ${{ steps.meta.outputs.tags }} 95 | labels: ${{ steps.meta.outputs.labels }} 96 | cache-from: type=gha 97 | target: ${{ matrix.type }} 98 | cache-to: type=gha,mode=max 99 | 100 | # Sign the resulting Docker image digest except on PRs. 101 | # This will only write to the public Rekor transparency log when the Docker 102 | # repository is public to avoid leaking data. If you would like to publish 103 | # transparency data even for private images, pass --force to cosign below. 104 | # https://github.com/sigstore/cosign 105 | - name: Sign the published Docker image 106 | if: ${{ github.event_name != 'pull_request' }} 107 | env: 108 | # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable 109 | TAGS: ${{ steps.meta.outputs.tags }} 110 | DIGEST: ${{ steps.build-and-push.outputs.digest }} 111 | # This step uses the identity token to provision an ephemeral certificate 112 | # against the sigstore community Fulcio instance. 113 | run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} 114 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | cmd/togomak/togomak 3 | cmd/togomak/*.hcl 4 | .togomak/pipelines/ 5 | examples/*/.togomak/ 6 | cmd/togomak/.togomak/ 7 | docs/book/ 8 | .terraform 9 | *.tfstate 10 | *.tfstate.backup 11 | *.tfplan 12 | *.out 13 | .terraform.lock.hcl 14 | coverage*.out 15 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | project_name: togomak 2 | builds: 3 | - main: ./cmd/togomak 4 | id: togomak 5 | env: [CGO_ENABLED=0] 6 | goos: 7 | - linux 8 | - windows 9 | - darwin 10 | goarch: 11 | - amd64 12 | - arm64 13 | 14 | release: 15 | prerelease: auto 16 | 17 | nfpms: 18 | - maintainer: Srevin Saju 19 | description: A CI/CD which works everywhere, even on your local environment. 20 | homepage: https://gitlab.com/srevinsaju/togomak 21 | license: MPL-2.0 22 | formats: 23 | - deb 24 | - rpm 25 | - apk 26 | 27 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [v2.0.0-alpha.16] 9 | - Add `stage.*.container.skip_workspace` boolean parameter to skip mounting the current working directory when using the docker plugin 10 | - Do not pass the PATH and host environment variables to the child docker container 11 | 12 | ## [v2.0.0-alpha.14] 13 | - Fixes `depends_on` not being respected on modules. 14 | 15 | ## [v2.0.0-alpha.13] 16 | - Changes the behavior of module lifecycles. By default all modules will be run if lifecycle.phase is unspecified. 17 | 18 | ## [v2.0.0-alpha.12] 19 | - Add `TOGOMAK_ARGS` environment variable. 20 | 21 | ## [v2.0.0-alpha.11] 22 | - Add `--logging.local.file` and `--logging.local.file.path` for writing logs to file. 23 | 24 | ## [v2.0.0-alpha.10] 25 | - Fix `path.module` incorrectly being populated for togomak modules 26 | 27 | ## [v2.0.0-alpha.9] 28 | - Add support for `modules` 29 | - Add JSON logger 30 | - Add `for_each` to modules 31 | - Add `variable` block 32 | 33 | ## [v2.0.0-alpha.4] 34 | - Add support for `for_each` meta argument 35 | 36 | ## [Older releases] 37 | > see `git log` for previous release history, or use GitHub releases page 38 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine AS builder 2 | 3 | WORKDIR /app 4 | COPY . . 5 | 6 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o /go/bin/togomak ./cmd/togomak 7 | 8 | 9 | FROM docker:cli AS docker 10 | RUN apk add --no-cache git 11 | COPY --from=builder /go/bin/togomak /usr/bin/togomak 12 | ENTRYPOINT ["/usr/bin/togomak"] 13 | 14 | 15 | FROM scratch AS tiny 16 | COPY --from=builder /go/bin/togomak /usr/bin/togomak 17 | ENTRYPOINT ["/usr/bin/togomak"] 18 | 19 | FROM alpine as alpine 20 | RUN apk add --no-cache git 21 | COPY --from=builder /go/bin/togomak /usr/bin/togomak 22 | ENTRYPOINT ["/usr/bin/togomak"] 23 | 24 | FROM gcr.io/cloud-builders/docker AS docker-buster 25 | COPY --from=builder /go/bin/togomak /usr/bin/togomak 26 | ENTRYPOINT ["/usr/bin/togomak"] 27 | 28 | FROM alpine as default 29 | -------------------------------------------------------------------------------- /cmd/togomak/utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/moby/sys/mountinfo" 5 | "github.com/spf13/afero" 6 | "github.com/srevinsaju/togomak/v1/internal/meta" 7 | "log" 8 | "path" 9 | "path/filepath" 10 | ) 11 | 12 | func autoDetectFilePath(cwd string) string { 13 | fs := afero.NewOsFs() 14 | absPath, err := filepath.Abs(cwd) 15 | if err != nil { 16 | panic(err) 17 | } 18 | p := path.Join(cwd, meta.ConfigFileName) 19 | exists, err := afero.Exists(fs, p) 20 | if err != nil { 21 | log.Fatal(err) 22 | } 23 | 24 | if exists { 25 | return p 26 | } 27 | p2 := path.Join(cwd, meta.BuildDirPrefix, meta.ConfigFileName) 28 | exists, err = afero.Exists(fs, p2) 29 | if err != nil { 30 | log.Fatal(err) 31 | } 32 | 33 | if exists { 34 | return p2 35 | } 36 | 37 | mountPoint, err := mountinfo.Mounted(absPath) 38 | if mountPoint { 39 | log.Fatalf("Couldn't find %s. Searched until %s", meta.ConfigFileName, absPath) 40 | } 41 | 42 | return autoDetectFilePath(path.Join(cwd, "..")) 43 | 44 | } 45 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Docs 2 | 3 | Moved to [srevinsaju/togomak-docs](https://github.com/srevinsaju/togomak-docs). 4 | For the live docs website, see [togomak.srev.in](https://togomak.srev.in) 5 | -------------------------------------------------------------------------------- /docs/img/screenshot-cover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srevinsaju/togomak/54334bd492d502b139492f63282323d6b9ee8c13/docs/img/screenshot-cover.png -------------------------------------------------------------------------------- /docs/img/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srevinsaju/togomak/54334bd492d502b139492f63282323d6b9ee8c13/docs/img/screenshot.png -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | The following are a list of examples. This list is autogenerated using `togomak --disable-concurrency` at [togomak](./togomak.hcl). 3 | 4 | ## Dynamic ANSI colors 5 | ANSI colors are provided as functions and variables 6 | which can be used to created colored terminal output 7 | platform agnostically. 8 | 9 | [Example](./ansi) 10 | 11 | ## Stage Conditions 12 | Make stage run conditionally based on `if` argument. 13 | 14 | [Example](./conditions) 15 | 16 | ## Demo of Togomak v1 Features 17 | Includes the most used features of togomak v1, previously on togomak v1 18 | `README.md` 19 | 20 | [Example](./demo) 21 | 22 | ## Customizing Docker Entrypoint 23 | This example provides a custom entrypoint for the docker 24 | container when natively calling them. 25 | 26 | [Example](./docker-entrypoint) 27 | 28 | ## Using Docker 29 | Run scripts in docker containers natively, from togomak. 30 | This example deals with mounting volumes and using the `ubuntu:latest` 31 | docker container. 32 | 33 | [Example](./docker) 34 | 35 | ## Environment Variables 36 | Derive environment variables from the host environment, and 37 | also pass them to stages using the `env {}` sub block. 38 | 39 | [Example](./env) 40 | 41 | ## Simple Stage 42 | Uses stage `args` argument to perform a simple `ls -al` 43 | 44 | [Example](./files) 45 | 46 | ## Using `for_each` 47 | This example uses a `for_each` on a stage using a `map`. 48 | Each item in the map specified as the argument to `for_each` 49 | will be iterated over and assigned `each.key` and `each.value`. 50 | 51 | [Example](./for-each-map) 52 | 53 | ## Using `for_each` on a module 54 | This is essentially the same example as `for_each`, but using 55 | this on `module` on a nested set array. 56 | In addition, this module also uses a remote github repository 57 | source for its module. 58 | 59 | [Example](./for-each-module) 60 | 61 | ## Function 62 | Demonstrates the use of default functions and string interpolation 63 | in stage scripts. This example deals with the `env()` helper function. 64 | 65 | [Example](./functions) 66 | 67 | ## Git 68 | Uses the `data.git` provider to clone a git repository and 69 | read contents of specific files to memory. 70 | 71 | [Example](./git) 72 | 73 | ## Git Tags 74 | same example as `git` but retrieves file from a specific tag. 75 | 76 | [Example](./git_tags) 77 | 78 | ## Stage hooks 79 | Runs a stage before or after the execution of a stage using 80 | `pre_hook` and `post_hook`. `post_hook` will run regardless of the stage execution statuses, 81 | which can be useful to check if a step failed or not. 82 | 83 | [Example](./hooks) 84 | 85 | ## Imports 86 | This example imports a subfolder and a remote repository togomak pipeline code 87 | and merges the caller pipeline with its contents to create a flattened pipeline. 88 | 89 | [Example](./import) 90 | 91 | ## Lifecycles 92 | Uses Togomak lifecycles to classify stages using `phases`. 93 | Consider reading [Lifecycle Usage](https://togomak.srev.in/docs/cli/usage) 94 | for a detailed explanation. 95 | 96 | [Example](./lifecycles) 97 | 98 | ## Local Blocks 99 | Uses a simple local block in a stage. 100 | 101 | [Example](./locals) 102 | 103 | ## Macros 104 | > **Note** 105 | > Consider using `module` instead. 106 | 107 | Invokes a stage with parameters as a macro, which can be reused, imported or customized with parameters. 108 | 109 | [Example](./macros) 110 | 111 | ## Using Modules with variable inputs 112 | This example loads a local module from a subfolder 113 | and passed it `local.*` values as inputs. 114 | This example is a module which does addition. 115 | 116 | [Example](./module-local) 117 | 118 | ## Simple Module 119 | A simple module from a local directory. 120 | 121 | [Example](./modules) 122 | 123 | ## Pre and Post steps 124 | Runs a step at the beginning, or the end of the pipeline, before 125 | and after all the stages, modules complete. 126 | 127 | [Example](./pre-post) 128 | 129 | ## Prompt 130 | Deprecated, use `variable {}` instead. 131 | Prompt data provider requests information from the user 132 | before the pipeline starts. This, however is done directly by 133 | the `variable {}` block, accessible through the `var.` 134 | attribute. 135 | 136 | [Example](./prompt) 137 | 138 | ## Using macros 139 | This example has a recursive, nested macro invocation. 140 | 141 | [Example](./recursive) 142 | 143 | ## Terraform 144 | Example on using Terraform data source blocks, using the `hashicorp/random` 145 | provider to create a `random_pet` name, and use them directly in your 146 | `togomak` pipeline, illustrating similar use cases where pipelines need 147 | to run against IP addresses of Terraform provisioned resources. 148 | 149 | [Example](./terraform) 150 | 151 | -------------------------------------------------------------------------------- /examples/ansi/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Dynamic ANSI colors 2 | description: | 3 | ANSI colors are provided as functions and variables 4 | which can be used to created colored terminal output 5 | platform agnostically. 6 | -------------------------------------------------------------------------------- /examples/ansi/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | stage "example" { 5 | name = "example" 6 | script = "echo ${ansifmt("green", "hello world in green")}" 7 | } 8 | -------------------------------------------------------------------------------- /examples/colors/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "colors" { 6 | for_each = toset([ 7 | "error", 8 | "success", 9 | "warn", 10 | "warning", 11 | "info", 12 | "red", 13 | "green", 14 | "blue", 15 | "yellow", 16 | "bold", 17 | "italic", 18 | "cyan", 19 | "grey", 20 | "white", 21 | "magenta", 22 | "orange", 23 | "hi-green", 24 | "hi-blue", 25 | "hi-magenta", 26 | "hi-black", 27 | "hi-white", 28 | "hi-red", 29 | "hi-cyan", 30 | "hi-yellow", 31 | ]) 32 | 33 | script = "echo ${ansifmt(each.key, "hello world")}" 34 | } 35 | -------------------------------------------------------------------------------- /examples/conditions/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Stage Conditions 2 | description: | 3 | Make stage run conditionally based on `if` argument. 4 | -------------------------------------------------------------------------------- /examples/conditions/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | data "env" "home" { 6 | key = "HOME" 7 | default = "@" 8 | } 9 | 10 | stage "example" { 11 | if = data.env.home.value != "@" 12 | name = "example" 13 | script = "echo hello world" 14 | } 15 | -------------------------------------------------------------------------------- /examples/demo/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Demo of Togomak v1 Features 2 | description: | 3 | Includes the most used features of togomak v1, previously on togomak v1 4 | `README.md` 5 | -------------------------------------------------------------------------------- /examples/demo/screenrecord.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srevinsaju/togomak/54334bd492d502b139492f63282323d6b9ee8c13/examples/demo/screenrecord.gif -------------------------------------------------------------------------------- /examples/demo/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | data "prompt" "repo_name" { 6 | prompt = "enter repo name: " 7 | default = "username/repo" 8 | } 9 | 10 | locals { 11 | repo = "srevinsaju/togomak" 12 | lint_tools = ["misspell", "golangci-lint", "abcgo"] 13 | build_types = ["amd64", "i386", "arm64"] 14 | } 15 | 16 | stage "lint" { 17 | script = <<-EOT 18 | echo 💅 running style checks for repo ${local.repo} 19 | %{for tool in local.lint_tools} 20 | echo "* running linter: ${tool}" 21 | sleep 1 22 | %{endfor} 23 | EOT 24 | } 25 | 26 | 27 | stage "build" { 28 | script = <<-EOT 29 | echo 👷 running ${ansifmt("green", "build")} 30 | %{for arch in local.build_types} 31 | echo "* building ${local.repo} for ${arch}..." 32 | sleep 1 33 | %{endfor} 34 | EOT 35 | } 36 | 37 | stage "deploy" { 38 | if = data.prompt.repo_name.value == "srevinsaju/togomak" 39 | depends_on = [stage.build] 40 | container { 41 | image = "hashicorp/terraform" 42 | } 43 | args = ["version"] 44 | } 45 | -------------------------------------------------------------------------------- /examples/docker-entrypoint/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Customizing Docker Entrypoint 2 | description: | 3 | This example provides a custom entrypoint for the docker 4 | container when natively calling them. 5 | 6 | -------------------------------------------------------------------------------- /examples/docker-entrypoint/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "apt" { 6 | container { 7 | image = "ubuntu:latest" 8 | entrypoint = ["apt"] 9 | } 10 | args = ["install"] 11 | } 12 | -------------------------------------------------------------------------------- /examples/docker/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Using Docker 2 | description: | 3 | Run scripts in docker containers natively, from togomak. 4 | This example deals with mounting volumes and using the `ubuntu:latest` 5 | docker container. 6 | -------------------------------------------------------------------------------- /examples/docker/diary/shinji.diary.txt: -------------------------------------------------------------------------------- 1 | another unfamiliar ceiling 2 | ~ shinji 3 | -------------------------------------------------------------------------------- /examples/docker/rei.diary.txt: -------------------------------------------------------------------------------- 1 | sometimes you need a little wishful thinking to keep on living 2 | ~ misato 3 | -------------------------------------------------------------------------------- /examples/docker/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "example" { 6 | container { 7 | image = "ubuntu" 8 | volume { 9 | source = "${cwd}/diary" 10 | destination = "/newdiary" 11 | } 12 | } 13 | script = <<-EOT 14 | #!/usr/bin/env bash 15 | ls -al 16 | for i in $(seq 1 10); do 17 | sleep 1 18 | echo "Loading $i..." 19 | done 20 | cat rei.diary.txt 21 | ls -al /newdiary 22 | EOT 23 | } 24 | -------------------------------------------------------------------------------- /examples/env/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Environment Variables 2 | description: | 3 | Derive environment variables from the host environment, and 4 | also pass them to stages using the `env {}` sub block. 5 | -------------------------------------------------------------------------------- /examples/env/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | data "env" "HOME" { 6 | key = "HOME" 7 | } 8 | 9 | stage "example" { 10 | name = "example" 11 | script = "echo ${data.env.HOME.value}" 12 | } 13 | 14 | stage "another" { 15 | env { 16 | name = "MY_NEW_HOME" 17 | value = data.env.HOME.value 18 | } 19 | script = "echo My new home is $MY_NEW_HOME" 20 | } 21 | -------------------------------------------------------------------------------- /examples/files/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Simple Stage 2 | description: | 3 | Uses stage `args` argument to perform a simple `ls -al` 4 | -------------------------------------------------------------------------------- /examples/files/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | 6 | stage "list" { 7 | name = "listing files" 8 | args = ["ls", "-al"] 9 | } 10 | -------------------------------------------------------------------------------- /examples/for-each-map/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Using `for_each` 2 | description: | 3 | This example uses a `for_each` on a stage using a `map`. 4 | Each item in the map specified as the argument to `for_each` 5 | will be iterated over and assigned `each.key` and `each.value`. 6 | 7 | -------------------------------------------------------------------------------- /examples/for-each-map/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | locals { 6 | m = { 7 | part1 = "You Are (Not) Alone." 8 | part2 = "You Can (Not) Advance." 9 | part3 = "You Can (Not) Redo." 10 | part3-1 = "Thrice Upon a Time." 11 | } 12 | } 13 | 14 | 15 | stage "movie" { 16 | for_each = local.m 17 | name = "example" 18 | script = <<-EOT 19 | echo "Evangelion ${each.key}: ${each.value}" 20 | EOT 21 | } 22 | -------------------------------------------------------------------------------- /examples/for-each-module/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Using `for_each` on a module 2 | description: | 3 | This is essentially the same example as `for_each`, but using 4 | this on `module` on a nested set array. 5 | In addition, this module also uses a remote github repository 6 | source for its module. 7 | -------------------------------------------------------------------------------- /examples/for-each-module/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | 6 | module "parallel" { 7 | for_each = toset(["alpha", "beta", "gamma"]) 8 | source = "github.com/srevinsaju/togomak-first-module" 9 | } 10 | -------------------------------------------------------------------------------- /examples/functions/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Function 2 | description: | 3 | Demonstrates the use of default functions and string interpolation 4 | in stage scripts. This example deals with the `env()` helper function. 5 | 6 | 7 | -------------------------------------------------------------------------------- /examples/functions/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | 6 | stage "home" { 7 | script = "echo ${env("HOME")}" 8 | } 9 | 10 | stage "non_existent_env" { 11 | script = "echo ${env("THIS_SHOULD_NOT_EXIST", "env does not exist")}" 12 | } 13 | -------------------------------------------------------------------------------- /examples/git/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Git 2 | description: | 3 | Uses the `data.git` provider to clone a git repository and 4 | read contents of specific files to memory. 5 | -------------------------------------------------------------------------------- /examples/git/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | data "git" "repo" { 6 | url = "https://github.com/srevinsaju/togomak" 7 | branch = "v1" 8 | files = ["togomak.hcl"] 9 | } 10 | 11 | stage "example" { 12 | name = "example" 13 | script = <<-EOT 14 | echo '${data.git.repo.files["togomak.hcl"]}' 15 | EOT 16 | } 17 | -------------------------------------------------------------------------------- /examples/git_tags/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Git Tags 2 | description: | 3 | same example as `git` but retrieves file from a specific tag. 4 | -------------------------------------------------------------------------------- /examples/git_tags/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | data "git" "repo" { 6 | url = "https://github.com/srevinsaju/togomak" 7 | files = ["togomak.hcl"] 8 | tag = "v1.2.0" 9 | } 10 | 11 | stage "example" { 12 | name = "example" 13 | script = <<-EOT 14 | echo '${data.git.repo.files["togomak.hcl"]}' 15 | EOT 16 | 17 | } 18 | -------------------------------------------------------------------------------- /examples/hooks/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Stage hooks 2 | description: | 3 | Runs a stage before or after the execution of a stage using 4 | `pre_hook` and `post_hook`. `post_hook` will run regardless of the stage execution statuses, 5 | which can be useful to check if a step failed or not. 6 | -------------------------------------------------------------------------------- /examples/hooks/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | 6 | stage "example" { 7 | script = "echo hello world" 8 | 9 | pre_hook { 10 | stage { 11 | script = "echo before the script for stage ${this.id} runs" 12 | } 13 | } 14 | 15 | post_hook { 16 | stage { 17 | script = "echo the script for ${this.id} done with status ${this.status}" 18 | } 19 | } 20 | } 21 | 22 | stage "example_2" { 23 | script = "echo bye world" 24 | 25 | pre_hook { 26 | stage { 27 | script = "echo before the script for stage ${this.id} runs" 28 | } 29 | } 30 | 31 | post_hook { 32 | stage { 33 | script = "echo the script for ${this.id} done with status ${this.status}" 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /examples/import/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Imports 2 | description: | 3 | This example imports a subfolder and a remote repository togomak pipeline code 4 | and merges the caller pipeline with its contents to create a flattened pipeline. 5 | -------------------------------------------------------------------------------- /examples/import/module/child/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | stage "child" { 5 | script = "echo hello world from the child module" 6 | } 7 | -------------------------------------------------------------------------------- /examples/import/module/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "another_stage" { 6 | script = "echo this is coming from module" 7 | } 8 | 9 | import { 10 | source = "./child" 11 | } 12 | -------------------------------------------------------------------------------- /examples/import/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | import { 6 | source = "./module" 7 | } 8 | 9 | stage "main" { 10 | script = "echo script from the main file" 11 | } 12 | 13 | import { 14 | source = "git::https://github.com/srevinsaju/togomak-first-module" 15 | } 16 | -------------------------------------------------------------------------------- /examples/lifecycles/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Lifecycles 2 | description: | 3 | Uses Togomak lifecycles to classify stages using `phases`. 4 | Consider reading [Lifecycle Usage](https://togomak.srev.in/docs/cli/usage) 5 | for a detailed explanation. 6 | -------------------------------------------------------------------------------- /examples/lifecycles/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "normal" { 6 | script = "echo hi" 7 | } 8 | 9 | stage "dont_execute" { 10 | if = false 11 | script = "echo this shouldnt be executed && exit 1" 12 | } 13 | 14 | stage "docker" { 15 | lifecycle { 16 | phase = ["build"] 17 | } 18 | script = "echo docker run ..." 19 | } 20 | 21 | stage "terraform_fmt_check" { 22 | lifecycle { 23 | phase = ["deploy", "default"] 24 | } 25 | script = "echo terraform ..." 26 | } 27 | 28 | stage "terraform" { 29 | depends_on = [stage.docker] 30 | lifecycle { 31 | phase = ["deploy"] 32 | } 33 | script = "echo terraform fmt -check ..." 34 | } 35 | 36 | -------------------------------------------------------------------------------- /examples/locals/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Local Blocks 2 | description: | 3 | Uses a simple local block in a stage. 4 | -------------------------------------------------------------------------------- /examples/locals/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | locals { 6 | nerv_headquarters = "Tokyo-3" 7 | pilot_name = "Shinji" 8 | } 9 | 10 | stage "eva01_synchronization" { 11 | name = "Eva-01 Synchronization Tests" 12 | script = "echo ${local.pilot_name} is now running synchronization tests at ${local.nerv_headquarters}" 13 | } 14 | -------------------------------------------------------------------------------- /examples/macros/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Macros 2 | description: | 3 | > **Note** 4 | > Consider using `module` instead. 5 | 6 | Invokes a stage with parameters as a macro, which can be reused, imported or customized with parameters. 7 | -------------------------------------------------------------------------------- /examples/macros/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | macro "explode" { 6 | stage "explode" { 7 | script = <<-EOT 8 | for i in $(seq 1 10); do 9 | sleep 0.1 10 | echo "${param.eva}: Loading $i..." 11 | done 12 | 13 | echo "${param.eva}: entry plug connected! pilot ${param.pilot} synchronized! 🤖" 14 | EOT 15 | } 16 | } 17 | 18 | 19 | stage "entry_plug_eva01" { 20 | use { 21 | macro = macro.explode 22 | parameters = { 23 | pilot = "Shinji Ikari 🙅‍♂️" 24 | eva = "01" 25 | } 26 | } 27 | } 28 | 29 | stage "entry_plug_eva02" { 30 | use { 31 | macro = macro.explode 32 | parameters = { 33 | pilot = "Asuka Langley Soryu 🙅‍♀️" 34 | eva = "02" 35 | } 36 | } 37 | } 38 | 39 | 40 | -------------------------------------------------------------------------------- /examples/module-local/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Using Modules with variable inputs 2 | description: | 3 | This example loads a local module from a subfolder 4 | and passed it `local.*` values as inputs. 5 | This example is a module which does addition. 6 | -------------------------------------------------------------------------------- /examples/module-local/calculator/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | variable "a" { 6 | type = number 7 | description = "first variable" 8 | } 9 | variable "b" { 10 | type = number 11 | description = "second variable" 12 | } 13 | 14 | variable "operation" { 15 | type = string 16 | description = "Operation to perform, any of: [add, subtract, multiply, divide]" 17 | } 18 | 19 | stage "add" { 20 | if = var.operation == "add" 21 | script = "echo ${var.a} + ${var.b} is ${var.a + var.b}" 22 | } 23 | 24 | stage "paths" { 25 | script = <<-EOT 26 | echo path.module: ${path.module} 27 | echo path.cwd: ${path.cwd} 28 | echo path.root: ${path.root} 29 | EOT 30 | } 31 | -------------------------------------------------------------------------------- /examples/module-local/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | locals { 6 | a = 99 7 | b = 1 8 | } 9 | 10 | stage "paths" { 11 | script = <<-EOT 12 | echo path.module: ${path.module} 13 | echo path.cwd: ${path.cwd} 14 | echo path.root: ${path.root} 15 | EOT 16 | } 17 | 18 | module "add" { 19 | source = "./calculator" 20 | a = local.a 21 | b = local.b 22 | operation = "add" 23 | } 24 | 25 | -------------------------------------------------------------------------------- /examples/module-phases-inheritance/calculator/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | variable "a" { 6 | type = number 7 | description = "first variable" 8 | } 9 | variable "b" { 10 | type = number 11 | description = "second variable" 12 | } 13 | 14 | variable "operation" { 15 | type = string 16 | description = "Operation to perform, any of: [add, subtract, multiply, divide]" 17 | } 18 | 19 | stage "add" { 20 | if = var.operation == "add" 21 | script = "echo ${var.a} + ${var.b} is ${var.a + var.b}" 22 | } 23 | 24 | stage "paths" { 25 | script = <<-EOT 26 | echo path.module: ${path.module} 27 | echo path.cwd: ${path.cwd} 28 | echo path.root: ${ansifmt("green", path.root)} 29 | EOT 30 | } 31 | -------------------------------------------------------------------------------- /examples/module-phases-inheritance/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | locals { 6 | a = 99 7 | b = 1 8 | } 9 | 10 | stage "paths" { 11 | script = <<-EOT 12 | echo path.module: ${path.module} 13 | echo path.cwd: ${path.cwd} 14 | echo path.root: ${path.root} 15 | EOT 16 | } 17 | 18 | module "add" { 19 | source = "./calculator" 20 | a = local.a 21 | b = local.b 22 | operation = "add" 23 | 24 | lifecycle { 25 | phase = ["add"] 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /examples/modules/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Simple Module 2 | description: | 3 | A simple module from a local directory. 4 | -------------------------------------------------------------------------------- /examples/modules/module/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | stage "example" { 5 | name = "example" 6 | script = "echo hello world" 7 | } 8 | -------------------------------------------------------------------------------- /examples/modules/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | 6 | module "something" { 7 | source = "./module" 8 | } 9 | 10 | 11 | stage "example" { 12 | script = "echo hello world from root" 13 | } 14 | -------------------------------------------------------------------------------- /examples/multiple-files/stage1.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "example" { 6 | name = "example" 7 | script = "echo hello world" 8 | } 9 | -------------------------------------------------------------------------------- /examples/multiple-files/stage2.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "example_2" { 6 | script = "echo bye world" 7 | } 8 | -------------------------------------------------------------------------------- /examples/multiple-files/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | -------------------------------------------------------------------------------- /examples/nested-filter/child/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "limited" { 6 | script = "echo limited" 7 | 8 | lifecycle { 9 | phase = ["build_phase"] 10 | } 11 | } 12 | 13 | stage "example" { 14 | name = "example" 15 | script = "echo hello world" 16 | } 17 | -------------------------------------------------------------------------------- /examples/nested-filter/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | module "example" { 6 | source = "./child" 7 | } 8 | -------------------------------------------------------------------------------- /examples/output/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "agent" { 6 | script = <<-EOT 7 | set -u 8 | echo "AGENT=Ryoji Kaji" >> $TOGOMAK_OUTPUTS 9 | EOT 10 | } 11 | 12 | stage "seele" { 13 | depends_on = [stage.agent] 14 | name = "seele" 15 | script = "echo The agent from Seele reporting! ${output.AGENT}" 16 | } 17 | -------------------------------------------------------------------------------- /examples/pre-post/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Pre and Post steps 2 | description: | 3 | Runs a step at the beginning, or the end of the pipeline, before 4 | and after all the stages, modules complete. 5 | -------------------------------------------------------------------------------- /examples/pre-post/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | pre { 6 | script = "echo first stage to be executed" 7 | } 8 | 9 | post { 10 | script = "echo last stage to be executed" 11 | } 12 | 13 | stage "example" { 14 | script = "echo hello world" 15 | } 16 | 17 | stage "example_2" { 18 | depends_on = [stage.example] 19 | script = "echo hello world 2" 20 | } 21 | -------------------------------------------------------------------------------- /examples/prompt/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Prompt 2 | description: | 3 | Deprecated, use `variable {}` instead. 4 | Prompt data provider requests information from the user 5 | before the pipeline starts. This, however is done directly by 6 | the `variable {}` block, accessible through the `var.` 7 | attribute. 8 | -------------------------------------------------------------------------------- /examples/prompt/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | data "env" "quit_if_not_shinji" { 6 | key = "QUIT_IF_NOT_SHINJI" 7 | default = "false" 8 | } 9 | 10 | data "prompt" "name" { 11 | prompt = "What is your name?" 12 | default = "Pen Pen" 13 | } 14 | 15 | stage "example" { 16 | name = "example" 17 | script = "echo hello ${data.prompt.name.value}" 18 | } 19 | 20 | stage "quit" { 21 | if = data.env.quit_if_not_shinji.value != "false" 22 | script = <<-EOT 23 | #!/usr/bin/env bash 24 | set -eux 25 | if [[ "${data.prompt.name.value}" != "Shinji Ikari" ]]; then 26 | echo "${data.prompt.name.value}" >> /tmp/quit 27 | exit 1 28 | fi 29 | EOT 30 | 31 | } 32 | -------------------------------------------------------------------------------- /examples/recursive/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Using macros 2 | description: | 3 | This example has a recursive, nested macro invocation. 4 | 5 | -------------------------------------------------------------------------------- /examples/recursive/bye/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "echo" { 6 | script = "echo Farewell, ${param.target}. Until we meet again in instrumetality!" 7 | } 8 | -------------------------------------------------------------------------------- /examples/recursive/hello/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "echo" { 6 | script = "echo Begin the synchronization! ${param.target} engage! 🚀" 7 | } 8 | -------------------------------------------------------------------------------- /examples/recursive/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | 6 | macro "hello" { 7 | source = "./hello" 8 | } 9 | macro "bye" { 10 | source = "./bye" 11 | } 12 | 13 | stage "hello_phase" { 14 | use { 15 | macro = macro.hello 16 | parameters = { 17 | target = "eva-01" 18 | } 19 | } 20 | } 21 | 22 | stage "bye_phase" { 23 | depends_on = [ 24 | stage.hello_phase 25 | ] 26 | use { 27 | macro = macro.bye 28 | parameters = { 29 | target = "world" 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /examples/remote-stages/.gitignore: -------------------------------------------------------------------------------- 1 | cmd/ 2 | -------------------------------------------------------------------------------- /examples/remote-stages/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | data "git" "eva01_source" { 6 | url = "https://github.com/srevinsaju/togomak" 7 | files = ["togomak.hcl"] 8 | } 9 | 10 | macro "gendo_brain" { 11 | files = data.git.eva01_source.files 12 | } 13 | 14 | stage "build_eva01" { 15 | name = "Building eva unit" 16 | use { 17 | macro = macro.gendo_brain 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /examples/terraform/.meta.yaml: -------------------------------------------------------------------------------- 1 | title: Terraform 2 | description: | 3 | Example on using Terraform data source blocks, using the `hashicorp/random` 4 | provider to create a `random_pet` name, and use them directly in your 5 | `togomak` pipeline, illustrating similar use cases where pipelines need 6 | to run against IP addresses of Terraform provisioned resources. 7 | -------------------------------------------------------------------------------- /examples/terraform/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/random" { 5 | version = "3.6.0" 6 | hashes = [ 7 | "h1:R5Ucn26riKIEijcsiOMBR3uOAjuOMfI1x7XvH4P6B1w=", 8 | "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", 9 | "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", 10 | "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", 11 | "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", 12 | "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", 13 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 14 | "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", 15 | "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", 16 | "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", 17 | "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", 18 | "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", 19 | "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /examples/terraform/random.tf: -------------------------------------------------------------------------------- 1 | resource "random_pet" "pet" { 2 | } 3 | 4 | -------------------------------------------------------------------------------- /examples/terraform/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | data "tf" "this" { 6 | source = "." 7 | allow_apply = true 8 | } 9 | 10 | 11 | stage "hello" { 12 | script = "echo hello world" 13 | } 14 | 15 | stage "delayed" { 16 | script = "echo Here is a random pet name: ${data.tf.this.random_pet.pet.id}" 17 | } 18 | -------------------------------------------------------------------------------- /examples/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | behavior { 4 | disable_concurrency = true 5 | } 6 | } 7 | 8 | locals { 9 | readme = "${path.module}/README.md" 10 | } 11 | 12 | 13 | stage "header" { 14 | script = <<-EOT 15 | rm ${local.readme} 16 | echo '# Examples' | tee -a ${local.readme} 17 | echo 'The following are a list of examples. This list is autogenerated using `togomak --disable-concurrency` at [togomak](./togomak.hcl).' | tee -a ${local.readme} 18 | echo '' | tee -a ${local.readme} 19 | EOT 20 | } 21 | 22 | stage "generate" { 23 | depends_on = [stage.header] 24 | for_each = fileset(".", "*/.meta.yaml") 25 | script = <<-EOT 26 | echo '## ${yamldecode(file(each.key)).title}' | tee -a ${local.readme} 27 | echo '${yamldecode(file(each.key)).description}' | tee -a ${local.readme} 28 | echo '[Example](./${dirname(each.key)})' | tee -a ${local.readme} 29 | echo '' | tee -a ${local.readme} 30 | EOT 31 | } 32 | -------------------------------------------------------------------------------- /examples/variables/calculator/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | variable "operation" { 6 | type = string 7 | description = "Possible operations: [add, subtract, multiply, divide]" 8 | } 9 | variable "a" { 10 | type = number 11 | } 12 | variable "b" { 13 | type = number 14 | } 15 | stage "add" { 16 | if = var.operation == "add" 17 | script = "echo sum of ${var.a} and ${var.b} is ${var.a + var.b}" 18 | } 19 | -------------------------------------------------------------------------------- /examples/variables/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | variable "name" { 6 | description = "Name of the individual executing this script (or bot)" 7 | } 8 | 9 | locals { 10 | numbers = toset([ 11 | { 12 | a = 30 13 | b = 40 14 | }, 15 | { 16 | a = 10 17 | b = 9 18 | }, 19 | { 20 | a = 99 21 | b = 1 22 | } 23 | ]) 24 | } 25 | 26 | stage "example" { 27 | name = "example" 28 | script = "echo hello world" 29 | } 30 | 31 | stage "welcome" { 32 | script = "echo hello ${var.name}" 33 | } 34 | 35 | stage "test" { 36 | for_each = local.numbers 37 | script = "echo ${each.value.a}, ${each.value.b}" 38 | } 39 | 40 | module "sum" { 41 | depends_on = [stage.example, stage.welcome] 42 | for_each = local.numbers 43 | source = "./calculator" 44 | 45 | a = each.value.a 46 | b = each.value.b 47 | operation = "add" 48 | } 49 | -------------------------------------------------------------------------------- /internal/behavior/models.go: -------------------------------------------------------------------------------- 1 | package behavior 2 | 3 | type Child struct { 4 | // Enabled is the flag to indicate whether the program is running in child mode 5 | Enabled bool 6 | 7 | // Parent is the flag to indicate whether the program is running in parent mode 8 | Parent string 9 | 10 | ParentLifecycles []string 11 | 12 | // ParentParams is the list of parameters to be passed to the parent 13 | ParentParams []string 14 | } 15 | 16 | type Behavior struct { 17 | initialized bool 18 | 19 | // Unattended is the flag to indicate whether the program is running in unattended mode 20 | Unattended bool 21 | 22 | // Ci is the flag to indicate whether the program is running in CI mode 23 | Ci bool 24 | 25 | // Child is the flag to indicate whether the program is running in child mode 26 | Child Child 27 | 28 | DryRun bool 29 | 30 | DisableConcurrency bool 31 | } 32 | 33 | func NewDefaultBehavior() *Behavior { 34 | return &Behavior{ 35 | Unattended: false, 36 | Ci: false, 37 | DryRun: true, 38 | Child: Child{ 39 | Enabled: false, 40 | Parent: "", 41 | ParentParams: []string{}, 42 | }, 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal/blocks/data/env.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/hashicorp/hcl/v2" 7 | "github.com/srevinsaju/togomak/v1/internal/conductor" 8 | "github.com/zclconf/go-cty/cty" 9 | "os" 10 | ) 11 | 12 | const ( 13 | EnvProviderAttrKey = "key" 14 | EnvProviderAttrDefault = "default" 15 | ) 16 | 17 | type EnvProvider struct { 18 | initialized bool 19 | Key hcl.Expression `hcl:"key" json:"key"` 20 | Default hcl.Expression `hcl:"default" json:"default"` 21 | 22 | keyParsed string 23 | def string 24 | defOk bool 25 | ctx context.Context 26 | } 27 | 28 | func (e *EnvProvider) Name() string { 29 | return "env" 30 | } 31 | 32 | func (e *EnvProvider) Initialized() bool { 33 | return e.initialized 34 | } 35 | 36 | func (e *EnvProvider) SetContext(context context.Context) { 37 | if !e.initialized { 38 | panic("provider not initialized") 39 | } 40 | e.ctx = context 41 | } 42 | 43 | func (e *EnvProvider) Version() string { 44 | return "1" 45 | } 46 | 47 | func (e *EnvProvider) New() Provider { 48 | return &EnvProvider{ 49 | initialized: true, 50 | } 51 | } 52 | 53 | func (e *EnvProvider) Attributes(conductor conductor.Conductor, ctx context.Context, id string, opts ...ProviderOption) (map[string]cty.Value, hcl.Diagnostics) { 54 | return map[string]cty.Value{ 55 | EnvProviderAttrKey: cty.StringVal(e.keyParsed), 56 | EnvProviderAttrDefault: cty.StringVal(e.def), 57 | }, nil 58 | } 59 | 60 | func (e *EnvProvider) Url() string { 61 | return "embedded::togomak.srev.in/providers/data/env" 62 | } 63 | 64 | func (e *EnvProvider) Schema() *hcl.BodySchema { 65 | schema := &hcl.BodySchema{ 66 | Attributes: []hcl.AttributeSchema{ 67 | { 68 | Name: EnvProviderAttrKey, 69 | Required: true, 70 | }, 71 | { 72 | Name: EnvProviderAttrDefault, 73 | Required: false, 74 | }, 75 | }, 76 | } 77 | return schema 78 | 79 | } 80 | 81 | func (e *EnvProvider) DecodeBody(conductor conductor.Conductor, body hcl.Body, opts ...ProviderOption) hcl.Diagnostics { 82 | if !e.initialized { 83 | panic("provider not initialized") 84 | } 85 | var diags hcl.Diagnostics 86 | hclContext := conductor.Eval().Context() 87 | 88 | schema := e.Schema() 89 | content, d := body.Content(schema) 90 | diags = diags.Extend(d) 91 | 92 | attr := content.Attributes["key"] 93 | var key cty.Value 94 | 95 | conductor.Eval().Mutex().RLock() 96 | key, d = attr.Expr.Value(hclContext) 97 | conductor.Eval().Mutex().RUnlock() 98 | diags = diags.Extend(d) 99 | 100 | e.keyParsed = key.AsString() 101 | 102 | attr, ok := content.Attributes["default"] 103 | if !ok { 104 | e.defOk = false 105 | e.def = "" 106 | return diags 107 | } 108 | 109 | conductor.Eval().Mutex().RLock() 110 | key, d = attr.Expr.Value(hclContext) 111 | conductor.Eval().Mutex().RUnlock() 112 | diags = diags.Extend(d) 113 | 114 | e.def = key.AsString() 115 | e.defOk = true 116 | 117 | return diags 118 | 119 | } 120 | 121 | func (e *EnvProvider) Value(conductor conductor.Conductor, ctx context.Context, id string, opts ...ProviderOption) (string, hcl.Diagnostics) { 122 | if !e.initialized { 123 | panic("provider not initialized") 124 | } 125 | v, exists := os.LookupEnv(e.keyParsed) 126 | if exists { 127 | return v, nil 128 | } 129 | if !e.defOk { 130 | return "", hcl.Diagnostics{ 131 | { 132 | Severity: hcl.DiagError, 133 | Summary: "environment variable not found", 134 | Detail: fmt.Sprintf("environment variable %s not found", e.keyParsed), 135 | }, 136 | } 137 | } 138 | return e.def, nil 139 | } 140 | -------------------------------------------------------------------------------- /internal/blocks/data/file.go: -------------------------------------------------------------------------------- 1 | package data 2 | -------------------------------------------------------------------------------- /internal/blocks/data/prompt.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/AlecAivazis/survey/v2" 7 | "github.com/hashicorp/hcl/v2" 8 | "github.com/srevinsaju/togomak/v1/internal/conductor" 9 | "github.com/srevinsaju/togomak/v1/internal/meta" 10 | "github.com/zclconf/go-cty/cty" 11 | "os" 12 | ) 13 | 14 | type PromptProvider struct { 15 | initialized bool 16 | 17 | Prompt hcl.Expression `hcl:"prompt" json:"prompt"` 18 | Default hcl.Expression `hcl:"default" json:"default"` 19 | 20 | promptParsed string 21 | def string 22 | ctx context.Context 23 | } 24 | 25 | func (e *PromptProvider) Name() string { 26 | return "prompt" 27 | } 28 | 29 | func (e *PromptProvider) SetContext(context context.Context) { 30 | e.ctx = context 31 | } 32 | 33 | func (e *PromptProvider) Version() string { 34 | return "1" 35 | } 36 | 37 | func (e *PromptProvider) Url() string { 38 | return "embedded::togomak.srev.in/providers/data/prompt" 39 | } 40 | 41 | func (e *PromptProvider) DecodeBody(conductor conductor.Conductor, body hcl.Body, opts ...ProviderOption) hcl.Diagnostics { 42 | if !e.initialized { 43 | panic("provider not initialized") 44 | } 45 | var diags hcl.Diagnostics 46 | 47 | hclContext := conductor.Eval().Context() 48 | 49 | schema := e.Schema() 50 | content, d := body.Content(schema) 51 | diags = append(diags, d...) 52 | 53 | attr := content.Attributes["prompt"] 54 | var key cty.Value 55 | 56 | conductor.Eval().Mutex().RLock() 57 | key, d = attr.Expr.Value(hclContext) 58 | conductor.Eval().Mutex().RUnlock() 59 | diags = append(diags, d...) 60 | 61 | e.promptParsed = key.AsString() 62 | 63 | attr = content.Attributes["default"] 64 | conductor.Eval().Mutex().RLock() 65 | key, d = attr.Expr.Value(hclContext) 66 | conductor.Eval().Mutex().RUnlock() 67 | diags = append(diags, d...) 68 | 69 | e.def = key.AsString() 70 | 71 | return diags 72 | 73 | } 74 | 75 | func (e *PromptProvider) New() Provider { 76 | return &PromptProvider{ 77 | initialized: true, 78 | } 79 | } 80 | 81 | func (e *PromptProvider) Schema() *hcl.BodySchema { 82 | return &hcl.BodySchema{ 83 | Attributes: []hcl.AttributeSchema{ 84 | { 85 | Name: "prompt", 86 | Required: false, 87 | }, 88 | { 89 | Name: "default", 90 | Required: false, 91 | }, 92 | }, 93 | } 94 | } 95 | 96 | func (e *PromptProvider) Attributes(conductor conductor.Conductor, ctx context.Context, id string, opts ...ProviderOption) (map[string]cty.Value, hcl.Diagnostics) { 97 | return map[string]cty.Value{ 98 | "prompt": cty.StringVal(e.promptParsed), 99 | "default": cty.StringVal(e.def), 100 | }, nil 101 | } 102 | 103 | func (e *PromptProvider) Initialized() bool { 104 | return e.initialized 105 | } 106 | 107 | func (e *PromptProvider) Value(conductor conductor.Conductor, ctx context.Context, id string, opts ...ProviderOption) (string, hcl.Diagnostics) { 108 | var diags hcl.Diagnostics 109 | if !e.initialized { 110 | panic("provider not initialized") 111 | } 112 | 113 | cfg := NewProviderConfig(opts...) 114 | 115 | logger := conductor.Logger().WithField("data", e.Name()) 116 | 117 | envVarName := fmt.Sprintf("%s%s__%s", meta.EnvVarPrefix, e.Name(), id) 118 | logger.Tracef("checking for environment variable %s", envVarName) 119 | envExists, ok := os.LookupEnv(envVarName) 120 | if ok { 121 | logger.Debug("environment variable found, using that") 122 | return envExists, nil 123 | } 124 | if cfg.Behavior.Unattended { 125 | logger.Warn("--unattended/--ci mode enabled, falling back to default") 126 | return e.def, nil 127 | } 128 | 129 | prompt := e.promptParsed 130 | if prompt == "" { 131 | prompt = fmt.Sprintf("Enter a value for %s:", e.Name()) 132 | } 133 | 134 | input := survey.Input{ 135 | Renderer: survey.Renderer{}, 136 | Message: prompt, 137 | Default: e.def, 138 | Help: "", 139 | Suggest: nil, 140 | } 141 | var resp string 142 | err := survey.AskOne(&input, &resp) 143 | if err != nil { 144 | logger.Warn("unable to get value from prompt: ", err) 145 | diags = append(diags, &hcl.Diagnostic{ 146 | Severity: hcl.DiagWarning, 147 | Summary: "unable to get value from prompt", 148 | Detail: err.Error(), 149 | }) 150 | return e.def, diags 151 | } 152 | 153 | return resp, diags 154 | } 155 | -------------------------------------------------------------------------------- /internal/blocks/data/provider.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "context" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/sirupsen/logrus" 7 | "github.com/srevinsaju/togomak/v1/internal/conductor" 8 | "github.com/zclconf/go-cty/cty" 9 | "sync" 10 | ) 11 | 12 | type Provider interface { 13 | Name() string 14 | Url() string 15 | Version() string 16 | Schema() *hcl.BodySchema 17 | Initialized() bool 18 | New() Provider 19 | 20 | SetContext(context context.Context) 21 | DecodeBody(conductor conductor.Conductor, body hcl.Body, opts ...ProviderOption) hcl.Diagnostics 22 | Value(conductor conductor.Conductor, ctx context.Context, id string, opts ...ProviderOption) (string, hcl.Diagnostics) 23 | Attributes(conductor conductor.Conductor, ctx context.Context, id string, opts ...ProviderOption) (map[string]cty.Value, hcl.Diagnostics) 24 | } 25 | 26 | type Eval struct { 27 | context *hcl.EvalContext 28 | mu *sync.RWMutex 29 | } 30 | 31 | func (e *Eval) Context() *hcl.EvalContext { 32 | return e.context 33 | } 34 | 35 | func (e *Eval) Mutex() *sync.RWMutex { 36 | return e.mu 37 | } 38 | 39 | type Conductor struct { 40 | eval *Eval 41 | logger *logrus.Entry 42 | } 43 | 44 | func (c *Conductor) Eval() *hcl.EvalContext { 45 | return c.eval.context 46 | } 47 | 48 | func (c *Conductor) Logger() *logrus.Entry { 49 | return c.logger 50 | } 51 | 52 | func Variables(e Provider, body hcl.Body) []hcl.Traversal { 53 | 54 | if !e.Initialized() { 55 | panic("provider not initialized") 56 | } 57 | var traversal []hcl.Traversal 58 | 59 | schema := e.Schema() 60 | d, _, diags := body.PartialContent(schema) 61 | if diags.HasErrors() { 62 | panic(diags.Error()) 63 | } 64 | for _, attr := range d.Attributes { 65 | traversal = append(traversal, attr.Expr.Variables()...) 66 | } 67 | return traversal 68 | } 69 | -------------------------------------------------------------------------------- /internal/blocks/data/provider_options.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "github.com/srevinsaju/togomak/v1/internal/behavior" 5 | "github.com/srevinsaju/togomak/v1/internal/path" 6 | ) 7 | 8 | type ProviderConfig struct { 9 | Paths *path.Path 10 | 11 | Behavior *behavior.Behavior 12 | } 13 | 14 | func NewDefaultProviderConfig() *ProviderConfig { 15 | return &ProviderConfig{ 16 | Paths: nil, 17 | } 18 | } 19 | 20 | type ProviderOption func(*ProviderConfig) 21 | 22 | func WithPaths(paths *path.Path) ProviderOption { 23 | return func(c *ProviderConfig) { 24 | c.Paths = paths 25 | } 26 | } 27 | 28 | func WithBehavior(behavior *behavior.Behavior) ProviderOption { 29 | return func(c *ProviderConfig) { 30 | c.Behavior = behavior 31 | } 32 | } 33 | 34 | func NewProviderConfig(opts ...ProviderOption) *ProviderConfig { 35 | c := NewDefaultProviderConfig() 36 | for _, opt := range opts { 37 | opt(c) 38 | } 39 | return c 40 | } 41 | -------------------------------------------------------------------------------- /internal/blocks/data/provider_stdlib.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import "fmt" 4 | 5 | var DefaultProviders = Providers{ 6 | &EnvProvider{}, 7 | &PromptProvider{}, 8 | // FileProvider{}, 9 | &GitProvider{}, 10 | &TfProvider{}, 11 | } 12 | 13 | type Providers []Provider 14 | 15 | func (p Providers) GoString() string { 16 | message := "" 17 | for _, provider := range p { 18 | message += fmt.Sprintf("%s, ", provider.Name()) 19 | } 20 | return message 21 | } 22 | 23 | func (p Providers) Get(name string) Provider { 24 | for _, provider := range p { 25 | if provider.Name() == name { 26 | return provider 27 | } 28 | } 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /internal/blocks/types.go: -------------------------------------------------------------------------------- 1 | package blocks 2 | 3 | const StageBlock = "stage" 4 | 5 | const LifecycleBlock = "lifecycle" 6 | 7 | const ModuleBlock = "module" 8 | 9 | const MacroBlock = "macro" 10 | 11 | const ParamBlock = "param" 12 | 13 | const VariableBlock = "variable" 14 | const VarBlock = "var" 15 | -------------------------------------------------------------------------------- /internal/c/context.go: -------------------------------------------------------------------------------- 1 | package c 2 | 3 | const ( 4 | TogomakContextPipeline = "pipeline" 5 | ) 6 | -------------------------------------------------------------------------------- /internal/c/context_test.go: -------------------------------------------------------------------------------- 1 | package c_test 2 | 3 | import "testing" 4 | 5 | func TestContext(t *testing.T) { 6 | return 7 | } 8 | -------------------------------------------------------------------------------- /internal/cache/clean.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | "github.com/srevinsaju/togomak/v1/internal/meta" 6 | "github.com/srevinsaju/togomak/v1/internal/x" 7 | "os" 8 | "path/filepath" 9 | "sync" 10 | ) 11 | 12 | func CleanCache(dir string, recursive bool) { 13 | dirPath := filepath.Join(dir, meta.BuildDirPrefix, "pipelines", "tmp") 14 | err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { 15 | if path == dirPath { 16 | return nil 17 | } 18 | if info != nil && info.IsDir() { 19 | fmt.Println("removing", path) 20 | x.Must(os.RemoveAll(path)) 21 | } 22 | return nil 23 | }) 24 | if err != nil { 25 | panic(err) 26 | } 27 | var wg sync.WaitGroup 28 | if recursive { 29 | entries, err := os.ReadDir(dir) 30 | if err != nil { 31 | panic(err) 32 | } 33 | for _, entry := range entries { 34 | 35 | if entry.IsDir() { 36 | wg.Add(1) 37 | go func(entry os.DirEntry) { 38 | defer wg.Done() 39 | CleanCache(filepath.Join(dir, entry.Name()), recursive) 40 | }(entry) 41 | } 42 | } 43 | } 44 | wg.Wait() 45 | 46 | } 47 | -------------------------------------------------------------------------------- /internal/ci/builder.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | const BuilderBlock = "togomak" 4 | 5 | type Behavior struct { 6 | DisableConcurrency bool `hcl:"disable_concurrency,optional" json:"disable_concurrency"` 7 | } 8 | 9 | type Builder struct { 10 | Version int `hcl:"version" json:"version"` 11 | Behavior *Behavior `hcl:"behavior,block" json:"behavior"` 12 | } 13 | -------------------------------------------------------------------------------- /internal/ci/conductor_config.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/srevinsaju/togomak/v1/internal/behavior" 5 | "github.com/srevinsaju/togomak/v1/internal/logging" 6 | "github.com/srevinsaju/togomak/v1/internal/path" 7 | "github.com/srevinsaju/togomak/v1/internal/rules" 8 | ) 9 | 10 | type ConfigPipeline struct { 11 | Filtered rules.Operations 12 | FilterQuery QueryEngines 13 | DryRun bool 14 | } 15 | 16 | type Interface struct { 17 | // Verbosity is the level of verbosity 18 | Verbosity int 19 | JSONLogging bool 20 | } 21 | 22 | type ConductorConfig struct { 23 | User string 24 | Hostname string 25 | 26 | Paths *path.Path 27 | 28 | Interface Interface 29 | 30 | // Pipeline is the pipeline configuration 31 | Pipeline ConfigPipeline 32 | 33 | // Behavior is the behavior of the program 34 | Behavior *behavior.Behavior 35 | 36 | Variables Variables 37 | 38 | Logging logging.Config 39 | } 40 | -------------------------------------------------------------------------------- /internal/ci/conductor_context.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | //ctx = context.WithValue(ctx, c.TogomakContextCi, cfg.Behavior.Ci) 4 | //ctx = context.WithValue(ctx, c.TogomakContextUnattended, cfg.Behavior.Unattended) 5 | //ctx = context.WithValue(ctx, c.TogomakContextLogger, logger) 6 | //ctx = context.WithValue(ctx, c.TogomakContextBootTime, time.Now()) 7 | //ctx = context.WithValue(ctx, c.TogomakContextPipelineId, pipelineId) 8 | //ctx = context.WithValue(ctx, c.TogomakContextOwd, cfg.Owd) 9 | //ctx = context.WithValue(ctx, c.TogomakContextCwd, cwd) 10 | //ctx = context.WithValue(ctx, c.TogomakContextHostname, cfg.Hostname) 11 | //ctx = context.WithValue(ctx, c.TogomakContextUsername, cfg.User) 12 | //ctx = context.WithValue(ctx, c.TogomakContextPipelineFilePath, cfg.Pipeline.FilePath) 13 | //ctx = context.WithValue(ctx, c.TogomakContextPipelineDryRun, cfg.Pipeline.DryRun) 14 | //ctx = context.WithValue(ctx, c.TogomakContextPipelineTmpDir, tempDir) 15 | //ctx = context.WithValue(ctx, c.TogomakContextHclDiagWriter, diagnosticTextWriter) 16 | -------------------------------------------------------------------------------- /internal/ci/context.go: -------------------------------------------------------------------------------- 1 | package ci 2 | -------------------------------------------------------------------------------- /internal/ci/daemon.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/srevinsaju/togomak/v1/internal/c" 7 | ) 8 | 9 | type DaemonLifecycleConfig struct { 10 | StopWhenComplete Blocks 11 | } 12 | 13 | type DaemonLifecycle struct { 14 | StopWhenComplete hcl.Expression `hcl:"stop_when_complete,optional" json:"stop_when_complete"` 15 | } 16 | 17 | func (l *DaemonLifecycle) Parse(ctx context.Context) (*DaemonLifecycleConfig, hcl.Diagnostics) { 18 | 19 | pipe := ctx.Value(c.TogomakContextPipeline).(*Pipeline) 20 | daemonLifecycle := &DaemonLifecycleConfig{} 21 | var diags hcl.Diagnostics 22 | 23 | if l == nil || l.StopWhenComplete == nil { 24 | return daemonLifecycle, diags 25 | } 26 | variables := l.StopWhenComplete.Variables() 27 | 28 | var runnableString []string 29 | for _, v := range variables { 30 | data, d := ResolveFromTraversal(v) 31 | diags = diags.Extend(d) 32 | if data == "" || diags.HasErrors() { 33 | continue 34 | } 35 | runnableString = append(runnableString, data) 36 | } 37 | var runnables Blocks 38 | for _, runnableId := range runnableString { 39 | runnable, diags := Resolve(pipe, runnableId) 40 | if diags.HasErrors() { 41 | return nil, diags 42 | } 43 | runnables = append(runnables, runnable) 44 | } 45 | daemonLifecycle.StopWhenComplete = runnables 46 | 47 | return daemonLifecycle, diags 48 | } 49 | 50 | func (l *DaemonLifecycle) Variables() []hcl.Traversal { 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /internal/ci/data.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | ) 6 | 7 | const DataBlock = "data" 8 | 9 | func (s *Data) Description() Description { 10 | return Description{Name: s.Name} 11 | } 12 | 13 | func (s *Data) Identifier() string { 14 | return s.Id 15 | } 16 | 17 | func (d Datas) ById(provider string, id string) (*Data, hcl.Diagnostics) { 18 | for _, data := range d { 19 | if data.Id == id && data.Provider == provider { 20 | return &data, nil 21 | } 22 | } 23 | return nil, hcl.Diagnostics{ 24 | { 25 | Severity: hcl.DiagError, 26 | Summary: "Data not found", 27 | Detail: "Data with id " + id + " not found", 28 | }, 29 | } 30 | } 31 | 32 | func (s *Data) Type() string { 33 | return DataBlock 34 | } 35 | 36 | func (s *Data) Set(k any, v any) { 37 | } 38 | 39 | func (s *Data) Get(k any) any { 40 | return nil 41 | } 42 | 43 | func (s *Data) IsDaemon() bool { 44 | return false 45 | } 46 | 47 | func (s *Data) Terminate(conductor *Conductor, safe bool) hcl.Diagnostics { 48 | return nil 49 | } 50 | 51 | func (s *Data) Kill() hcl.Diagnostics { 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /internal/ci/data_hcl.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | dataBlock "github.com/srevinsaju/togomak/v1/internal/blocks/data" 6 | ) 7 | 8 | func (s *Data) Variables() []hcl.Traversal { 9 | var traversal []hcl.Traversal 10 | provider := dataBlock.DefaultProviders.Get(s.Provider) 11 | // TODO: this will panic, if the provider is not found 12 | if provider == nil { 13 | panic("provider not found") 14 | } 15 | provide := provider.New() 16 | traversal = append(traversal, dataBlock.Variables(provide, s.Body)...) 17 | return traversal 18 | } 19 | 20 | func (d Datas) Variables() []hcl.Traversal { 21 | var traversal []hcl.Traversal 22 | for _, data := range d { 23 | traversal = append(traversal, data.Variables()...) 24 | } 25 | return traversal 26 | } 27 | -------------------------------------------------------------------------------- /internal/ci/data_lifecycle.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "github.com/hashicorp/hcl/v2" 6 | ) 7 | 8 | func (s *Data) ExecutionOptions(ctx context.Context) (*DaemonLifecycleConfig, hcl.Diagnostics) { 9 | return nil, nil 10 | } 11 | -------------------------------------------------------------------------------- /internal/ci/data_logging.go: -------------------------------------------------------------------------------- 1 | package ci 2 | -------------------------------------------------------------------------------- /internal/ci/data_prop.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | func (s *Data) Override() bool { 6 | return false 7 | } 8 | 9 | // CheckIfDistinct checks if the data in d and dd are distinct 10 | // TODO: check if this is a good way to do this 11 | func (d Datas) CheckIfDistinct(dd Datas) hcl.Diagnostics { 12 | var diags hcl.Diagnostics 13 | for _, block := range d { 14 | for _, block2 := range dd { 15 | if block.Id == block2.Id { 16 | diags = append(diags, &hcl.Diagnostic{ 17 | Severity: hcl.DiagError, 18 | Summary: "Duplicate data block", 19 | Detail: "Data with id " + block.Id + " is defined more than once", 20 | }) 21 | } 22 | } 23 | } 24 | return diags 25 | } 26 | -------------------------------------------------------------------------------- /internal/ci/data_prop_test.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestData_Override(t *testing.T) { 9 | assert.Equal(t, (&Data{}).Override(), false) 10 | } 11 | 12 | func TestDatas_CheckIfDistinct(t *testing.T) { 13 | data1 := Datas{ 14 | Data{ 15 | Id: "data1", 16 | }, 17 | Data{ 18 | Id: "data2", 19 | }, 20 | } 21 | data2 := Datas{ 22 | Data{ 23 | Id: "data3", 24 | }, 25 | Data{ 26 | Id: "data4", 27 | }, 28 | } 29 | 30 | assert.Equal(t, data1.CheckIfDistinct(data2).HasErrors(), false) 31 | assert.Equal(t, data1.CheckIfDistinct(data1).HasErrors(), true) 32 | } 33 | -------------------------------------------------------------------------------- /internal/ci/data_provider.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | const DataProviderBlock = "provider" 6 | 7 | type DataProvider struct { 8 | Name string `hcl:"name,label" json:"name"` 9 | Url string `hcl:"url,optional" json:"url,omitempty"` 10 | } 11 | 12 | func (d DataProvider) Variables() []hcl.Traversal { 13 | return nil 14 | } 15 | 16 | type DataProviders []DataProvider 17 | 18 | func (d DataProviders) Variables() []hcl.Traversal { 19 | var traversal []hcl.Traversal 20 | for _, provider := range d { 21 | traversal = append(traversal, provider.Variables()...) 22 | } 23 | return traversal 24 | } 25 | -------------------------------------------------------------------------------- /internal/ci/data_retry.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | func (s *Data) CanRetry() bool { 4 | return false 5 | } 6 | 7 | func (s *Data) MaxRetries() int { 8 | return 0 9 | } 10 | 11 | func (s *Data) MinRetryBackoff() int { 12 | return 0 13 | } 14 | func (s *Data) MaxRetryBackoff() int { 15 | return 0 16 | } 17 | 18 | func (s *Data) RetryExponentialBackoff() bool { 19 | return false 20 | 21 | } 22 | -------------------------------------------------------------------------------- /internal/ci/data_retry_test.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "testing" 4 | 5 | func TestData_CanRetry(t *testing.T) { 6 | data := Data{} 7 | if data.CanRetry() { 8 | t.Error("CanRetry() should return false") 9 | } 10 | } 11 | 12 | func TestData_Description(t *testing.T) { 13 | data := Data{} 14 | if data.Description().Description != "" { 15 | t.Error("Description() should return empty string") 16 | } 17 | } 18 | 19 | func TestData_MaxRetries(t *testing.T) { 20 | data := Data{} 21 | if data.MaxRetries() != 0 { 22 | t.Error("MaxRetries() should return 0") 23 | } 24 | } 25 | 26 | func TestData_MaxRetryBackoff(t *testing.T) { 27 | data := Data{} 28 | if data.MaxRetryBackoff() != 0 { 29 | t.Error("MaxRetryBackoff() should return 0") 30 | } 31 | } 32 | 33 | func TestData_MinRetryBackoff(t *testing.T) { 34 | data := Data{} 35 | if data.MinRetryBackoff() != 0 { 36 | t.Error("MinRetryBackoff() should return 0") 37 | } 38 | } 39 | 40 | func TestData_RetryExponentialBackoff(t *testing.T) { 41 | data := Data{} 42 | if data.RetryExponentialBackoff() { 43 | t.Error("RetryExponentialBackoff() should return false") 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /internal/ci/data_run.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/hcl/v2" 6 | dataBlock "github.com/srevinsaju/togomak/v1/internal/blocks/data" 7 | "github.com/srevinsaju/togomak/v1/internal/global" 8 | "github.com/srevinsaju/togomak/v1/internal/runnable" 9 | "github.com/zclconf/go-cty/cty" 10 | ) 11 | 12 | const ( 13 | DataAttrValue = "value" 14 | ) 15 | 16 | func (s *Data) Prepare(conductor *Conductor, skip bool, overridden bool) hcl.Diagnostics { 17 | return nil // no-op 18 | } 19 | 20 | func (s *Data) Run(conductor *Conductor, options ...runnable.Option) (diags hcl.Diagnostics) { 21 | logger := conductor.Logger().WithField("data", s.Id) 22 | logger.Debugf("running %s.%s.%s", DataBlock, s.Provider, s.Id) 23 | hclContext := conductor.Eval().Context() 24 | ctx := conductor.Context() 25 | 26 | var d hcl.Diagnostics 27 | 28 | cfg := runnable.NewConfig(options...) 29 | opts := []dataBlock.ProviderOption{ 30 | dataBlock.WithPaths(cfg.Paths), 31 | dataBlock.WithBehavior(cfg.Behavior), 32 | } 33 | 34 | // region: mutating the data map 35 | // TODO: move it to a dedicated helper function 36 | 37 | // -> update r.Value accordingly 38 | var validProvider bool 39 | var value string 40 | var attr map[string]cty.Value 41 | for _, pr := range dataBlock.DefaultProviders { 42 | if pr.Name() == s.Provider { 43 | validProvider = true 44 | provide := pr.New() 45 | provide.SetContext(ctx) 46 | diags = diags.Extend(provide.DecodeBody(conductor, s.Body, opts...)) 47 | value, d = provide.Value(conductor, ctx, s.Id, opts...) 48 | diags = diags.Extend(d) 49 | attr, d = provide.Attributes(conductor, ctx, s.Id, opts...) 50 | diags = diags.Extend(d) 51 | break 52 | } 53 | } 54 | if !validProvider { 55 | diags = diags.Append(&hcl.Diagnostic{ 56 | Severity: hcl.DiagError, 57 | Summary: fmt.Sprintf("invalid provider %s", s.Provider), 58 | Detail: fmt.Sprintf("built-in providers are %s", dataBlock.DefaultProviders), 59 | }) 60 | return diags 61 | } 62 | 63 | if diags.HasErrors() { 64 | return diags 65 | } 66 | 67 | m := make(map[string]cty.Value) 68 | m[DataAttrValue] = cty.StringVal(value) 69 | for k, v := range attr { 70 | m[k] = v 71 | } 72 | 73 | global.DataBlockEvalContextMutex.Lock() 74 | 75 | conductor.Eval().Mutex().RLock() 76 | 77 | data := hclContext.Variables[DataBlock] 78 | 79 | var dataMutated map[string]cty.Value 80 | if data.IsNull() { 81 | dataMutated = make(map[string]cty.Value) 82 | } else { 83 | dataMutated = data.AsValueMap() 84 | } 85 | provider := dataMutated[s.Provider] 86 | var providerMutated map[string]cty.Value 87 | if provider.IsNull() { 88 | providerMutated = make(map[string]cty.Value) 89 | } else { 90 | providerMutated = provider.AsValueMap() 91 | } 92 | providerMutated[s.Id] = cty.ObjectVal(m) 93 | dataMutated[s.Provider] = cty.ObjectVal(providerMutated) 94 | conductor.Eval().Mutex().RUnlock() 95 | 96 | conductor.Eval().Mutex().Lock() 97 | hclContext.Variables[DataBlock] = cty.ObjectVal(dataMutated) 98 | conductor.Eval().Mutex().Unlock() 99 | 100 | global.DataBlockEvalContextMutex.Unlock() 101 | // endregion 102 | 103 | if diags.HasErrors() { 104 | return diags 105 | } 106 | 107 | v := s.Variables() 108 | logger.Debug(fmt.Sprintf("%s variables: %v", s.Identifier(), v)) 109 | return nil 110 | } 111 | 112 | func (s *Data) CanRun(conductor *Conductor, options ...runnable.Option) (bool, hcl.Diagnostics) { 113 | return true, nil 114 | } 115 | 116 | func (s *Data) Terminated() bool { 117 | return true 118 | } 119 | -------------------------------------------------------------------------------- /internal/ci/data_schema.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | type Data struct { 6 | Provider string `hcl:"provider,label" json:"provider"` 7 | Id string `hcl:"id,label" json:"id"` 8 | 9 | Name string `hcl:"name,optional" json:"name"` 10 | Value string `json:"value"` 11 | 12 | Body hcl.Body `hcl:",remain"` 13 | } 14 | 15 | type Datas []Data 16 | -------------------------------------------------------------------------------- /internal/ci/data_test.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestData_Kill(t *testing.T) { 9 | data := Data{} 10 | if data.Kill() != nil { 11 | t.Error("Kill() should do nothing") 12 | } 13 | } 14 | 15 | func TestData_Terminate(t *testing.T) { 16 | data := Data{} 17 | if data.Terminate(nil, false) != nil { 18 | t.Error("Terminate() should do nothing") 19 | } 20 | } 21 | 22 | func TestData_Type(t *testing.T) { 23 | data := Data{} 24 | if data.Type() != DataBlock { 25 | t.Error("Type() should return data") 26 | } 27 | } 28 | 29 | func TestData_IsDaemon(t *testing.T) { 30 | data := Data{} 31 | if data.IsDaemon() { 32 | t.Error("IsDaemon() should return false") 33 | } 34 | } 35 | 36 | func TestData_Set(t *testing.T) { 37 | data := Data{} 38 | data.Set("key", "value") 39 | } 40 | 41 | func TestData_Get(t *testing.T) { 42 | data := Data{} 43 | assert.Equal(t, data.Get("key"), nil) 44 | } 45 | -------------------------------------------------------------------------------- /internal/ci/graph.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/kendru/darwin/go/depgraph" 7 | "github.com/srevinsaju/togomak/v1/internal/blocks" 8 | "github.com/srevinsaju/togomak/v1/internal/meta" 9 | "github.com/srevinsaju/togomak/v1/internal/x" 10 | ) 11 | 12 | func GraphResolve(ctx context.Context, pipe *Pipeline, g *depgraph.Graph, v []hcl.Traversal, child string) hcl.Diagnostics { 13 | var diags hcl.Diagnostics 14 | 15 | _, d := Resolve(pipe, child) 16 | diags = diags.Extend(d) 17 | if diags.HasErrors() { 18 | return diags 19 | } 20 | 21 | for _, variable := range v { 22 | parent, d := ResolveFromTraversal(variable) 23 | diags = diags.Extend(d) 24 | if parent == "" { 25 | continue 26 | } 27 | 28 | _, d = Resolve(pipe, parent) 29 | diags = diags.Extend(d) 30 | err := g.DependOn(child, parent) 31 | 32 | if err != nil { 33 | diags = diags.Append(&hcl.Diagnostic{ 34 | Severity: hcl.DiagError, 35 | Summary: "Invalid dependency", 36 | Detail: err.Error(), 37 | }) 38 | } 39 | 40 | } 41 | return diags 42 | } 43 | func GraphTopoSort(conductor *Conductor, pipe *Pipeline) (*depgraph.Graph, hcl.Diagnostics) { 44 | g := depgraph.New() 45 | var diags hcl.Diagnostics 46 | ctx := conductor.Context() 47 | logger := conductor.Logger().WithField("orchestra", "graph") 48 | 49 | x.Must(g.DependOn(meta.PreStage, meta.RootStage)) 50 | x.Must(g.DependOn(meta.PostStage, meta.PreStage)) 51 | 52 | for _, local := range pipe.Local { 53 | self := x.RenderBlock(LocalBlock, local.Key) 54 | err := g.DependOn(self, meta.RootStage) 55 | if err != nil { 56 | panic(err) 57 | } 58 | 59 | v := local.Variables() 60 | d := GraphResolve(ctx, pipe, g, v, self) 61 | diags = diags.Extend(d) 62 | } 63 | 64 | for _, macro := range pipe.Macros { 65 | self := x.RenderBlock(blocks.MacroBlock, macro.Id) 66 | err := g.DependOn(self, meta.RootStage) 67 | // the addition of the root stage is to ensure that the macro block is always executed 68 | // before any stage 69 | // this function should succeed always 70 | if err != nil { 71 | panic(err) 72 | } 73 | 74 | v := macro.Variables() 75 | d := GraphResolve(ctx, pipe, g, v, self) 76 | diags = diags.Extend(d) 77 | } 78 | for _, data := range pipe.Data { 79 | self := x.RenderBlock(DataBlock, data.Provider, data.Id) 80 | err := g.DependOn(self, meta.RootStage) 81 | // the addition of the root stage is to ensure that the data block is always executed 82 | // before any stage 83 | // this function should succeed always 84 | if err != nil { 85 | panic(err) 86 | } 87 | 88 | // all pre-stage blocks depend on the data block 89 | err = g.DependOn(meta.PreStage, self) 90 | if err != nil { 91 | panic(err) 92 | } 93 | 94 | v := data.Variables() 95 | d := GraphResolve(ctx, pipe, g, v, self) 96 | diags = diags.Extend(d) 97 | } 98 | 99 | for _, block := range pipe.Vars { 100 | self := x.RenderBlock(blocks.VarBlock, block.Id) 101 | err := g.DependOn(self, meta.RootStage) 102 | // the addition of the root stage is to ensure that the var block is always executed 103 | // before any stage 104 | // this function should succeed always 105 | if err != nil { 106 | panic(err) 107 | } 108 | 109 | // all pre-stage blocks depend on the data block 110 | err = g.DependOn(meta.PreStage, self) 111 | if err != nil { 112 | panic(err) 113 | } 114 | 115 | v := block.Variables() 116 | d := GraphResolve(ctx, pipe, g, v, self) 117 | diags = diags.Extend(d) 118 | } 119 | 120 | for _, stage := range pipe.Stages { 121 | self := x.RenderBlock(blocks.StageBlock, stage.Id) 122 | err := g.DependOn(self, meta.PreStage) 123 | if err != nil { 124 | panic(err) 125 | } 126 | 127 | err = g.DependOn(meta.PostStage, self) 128 | if err != nil { 129 | panic(err) 130 | } 131 | 132 | v := stage.Variables() 133 | d := GraphResolve(ctx, pipe, g, v, self) 134 | diags = diags.Extend(d) 135 | } 136 | 137 | for _, module := range pipe.Modules { 138 | self := x.RenderBlock(blocks.ModuleBlock, module.Id) 139 | err := g.DependOn(self, meta.PreStage) 140 | if err != nil { 141 | panic(err) 142 | } 143 | 144 | err = g.DependOn(meta.PostStage, self) 145 | if err != nil { 146 | panic(err) 147 | } 148 | 149 | v := module.Variables() 150 | d := GraphResolve(ctx, pipe, g, v, self) 151 | diags = diags.Extend(d) 152 | } 153 | 154 | for i, layer := range g.TopoSortedLayers() { 155 | logger.Debugf("layer %d: %s", i, layer) 156 | } 157 | return g, diags 158 | 159 | } 160 | -------------------------------------------------------------------------------- /internal/ci/hooks.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | func (s *PreStage) ToStage() *Stage { 4 | return &Stage{Id: "togomak.pre", CoreStage: s.CoreStage} 5 | } 6 | 7 | func (s *PostStage) ToStage() *Stage { 8 | return &Stage{Id: "togomak.post", CoreStage: s.CoreStage} 9 | } 10 | -------------------------------------------------------------------------------- /internal/ci/import.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/zclconf/go-cty/cty" 6 | ) 7 | 8 | const ImportBlock = "import" 9 | 10 | func (m *Import) Description() Description { 11 | return Description{Name: m.id} 12 | } 13 | 14 | func (m *Import) Identifier() string { 15 | if m.id == "" { 16 | panic("id not set") 17 | } 18 | return m.id 19 | } 20 | 21 | func (i Imports) ById(id string) (*Import, hcl.Diagnostics) { 22 | for _, macro := range i { 23 | if macro.Identifier() == id { 24 | return macro, nil 25 | } 26 | } 27 | return nil, hcl.Diagnostics{ 28 | { 29 | Severity: hcl.DiagError, 30 | Summary: "import not found", 31 | Detail: "import with id " + id + " not found", 32 | }, 33 | } 34 | } 35 | 36 | func (i Imports) PopulateProperties(conductor *Conductor) hcl.Diagnostics { 37 | var diags hcl.Diagnostics 38 | for _, imp := range i { 39 | diags = diags.Extend(imp.populateProperties(conductor)) 40 | } 41 | return diags 42 | } 43 | 44 | func (m *Import) Type() string { 45 | return ImportBlock 46 | } 47 | 48 | func (m *Import) Variables() []hcl.Traversal { 49 | return nil 50 | } 51 | 52 | func (m *Import) IsDaemon() bool { 53 | return false 54 | } 55 | 56 | func (m *Import) Terminate(safe bool) hcl.Diagnostics { 57 | return nil 58 | } 59 | 60 | func (m *Import) Kill() hcl.Diagnostics { 61 | return nil 62 | } 63 | 64 | func (m *Import) Set(k any, v any) { 65 | } 66 | 67 | func (m *Import) Get(k any) any { 68 | return nil 69 | } 70 | 71 | func (m *Import) populateProperties(conductor *Conductor) hcl.Diagnostics { 72 | evalContext := conductor.Eval().Context() 73 | conductor.Eval().Mutex().RLock() 74 | s, diags := m.Source.Value(evalContext) 75 | conductor.Eval().Mutex().RUnlock() 76 | 77 | if s.Type() != cty.String { 78 | diags = diags.Append(&hcl.Diagnostic{ 79 | Severity: hcl.DiagError, 80 | Summary: "invalid source", 81 | Detail: "source should be a string", 82 | }) 83 | return diags 84 | } 85 | 86 | if diags.HasErrors() { 87 | return diags 88 | } 89 | m.id = s.AsString() 90 | return diags 91 | } 92 | -------------------------------------------------------------------------------- /internal/ci/import_expand.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "crypto/sha256" 5 | "fmt" 6 | "github.com/hashicorp/go-getter" 7 | "github.com/hashicorp/hcl/v2" 8 | "github.com/srevinsaju/togomak/v1/internal/ui" 9 | "path/filepath" 10 | ) 11 | 12 | func (m *Import) Expand(conductor *Conductor, pwd string, dst string) (*Pipeline, hcl.Diagnostics) { 13 | logger := conductor.Logger().WithField("import", "") 14 | var diags hcl.Diagnostics 15 | shaIdentifier := sha256.Sum256([]byte(m.Identifier())) 16 | clientImportPath, err := filepath.Abs(filepath.Join(dst, fmt.Sprintf("%x", shaIdentifier))) 17 | if err != nil { 18 | panic(err) 19 | } 20 | 21 | // fmt.Println(pwd, dst, m.Source, fmt.Sprintf("%x", shaIdentifier)) 22 | get := getter.Client{ 23 | Ctx: conductor.Context(), 24 | Src: m.Identifier(), 25 | Dir: true, 26 | Pwd: pwd, 27 | Dst: clientImportPath, 28 | } 29 | ppb := ui.NewPassiveProgressBar(logger, fmt.Sprintf("pulling %s", m.Identifier())) 30 | ppb.Init() 31 | err = get.Get() 32 | if err != nil { 33 | diags = diags.Append(&hcl.Diagnostic{ 34 | Severity: hcl.DiagError, 35 | Summary: "import failed", 36 | Detail: fmt.Sprintf("import of %s failed: %s", m.Identifier(), err.Error()), 37 | }) 38 | } 39 | ppb.Done() 40 | 41 | p, d := ReadDirFromPath(conductor, clientImportPath) 42 | diags = diags.Extend(d) 43 | if diags.HasErrors() { 44 | return nil, diags 45 | } 46 | if p.Imports != nil { 47 | d := p.Imports.PopulateProperties(conductor) 48 | if d.HasErrors() { 49 | return nil, d 50 | } 51 | p, d = p.ExpandImports(conductor, clientImportPath) 52 | diags = diags.Extend(d) 53 | } 54 | return p, diags 55 | 56 | } 57 | -------------------------------------------------------------------------------- /internal/ci/import_schema.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | type Import struct { 6 | id string 7 | Source hcl.Expression `hcl:"source" json:"source"` 8 | } 9 | 10 | type Imports []*Import 11 | -------------------------------------------------------------------------------- /internal/ci/lifecycle.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | type LifecycleType int64 4 | 5 | // Lifecycle is inspired from the Maven build lifecycles" 6 | // validate - validate the project is correct and all necessary information is available 7 | // compile - compile the source code of the project 8 | // test - test the compiled source code using a suitable unit testing framework. These tests should not require the code be packaged or deployed 9 | // package - take the compiled code and package it in its distributable format, such as a JAR. 10 | // verify - run any checks on results of integration tests to ensure quality criteria are met 11 | // install - install the package into the local repository, for use as a dependency in other projects locally 12 | // deploy - done in the build environment, copies the final package to the remote repository for sharing with other developers and projects. 13 | 14 | const ( 15 | LifecycleDefault LifecycleType = iota 16 | 17 | LifecycleValidate 18 | LifecycleCompile 19 | LifecycleTest 20 | LifecyclePackage 21 | LifecycleVerify 22 | LifecycleInstall 23 | LifecycleDeploy 24 | 25 | LifecycleInvalid = -1 26 | ) 27 | 28 | func (ty LifecycleType) String() string { 29 | switch ty { 30 | case LifecycleDefault: 31 | return "default" 32 | case LifecycleValidate: 33 | return "validate" 34 | case LifecycleCompile: 35 | return "compile" 36 | case LifecycleTest: 37 | return "test" 38 | case LifecyclePackage: 39 | return "package" 40 | case LifecycleVerify: 41 | return "verify" 42 | case LifecycleInstall: 43 | return "install" 44 | case LifecycleDeploy: 45 | return "deploy" 46 | } 47 | panic("invalid lifecycle type") 48 | } 49 | 50 | func LifecycleUnmarshall(v string) (LifecycleType, bool) { 51 | lifecycle := []LifecycleType{ 52 | LifecycleDefault, 53 | LifecycleValidate, 54 | LifecycleCompile, 55 | LifecycleTest, 56 | LifecyclePackage, 57 | LifecycleVerify, 58 | LifecycleDeploy, 59 | } 60 | for _, ly := range lifecycle { 61 | if ly.String() == v { 62 | return ly, true 63 | } 64 | } 65 | return LifecycleInvalid, false 66 | 67 | } 68 | -------------------------------------------------------------------------------- /internal/ci/local_logging.go: -------------------------------------------------------------------------------- 1 | package ci 2 | -------------------------------------------------------------------------------- /internal/ci/locals.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/hcl/v2" 6 | ) 7 | 8 | const LocalsBlock = "locals" 9 | const LocalBlock = "local" 10 | 11 | func (l Locals) Description() Description { 12 | return Description{} 13 | } 14 | 15 | func (l Locals) Identifier() string { 16 | return fmt.Sprintf("locals.%p", &l) 17 | } 18 | 19 | func (l Locals) Type() string { 20 | return LocalsBlock 21 | } 22 | 23 | func (l Locals) Expand() ([]*Local, hcl.Diagnostics) { 24 | var locals []*Local 25 | var diags hcl.Diagnostics 26 | attr, err := l.Body.JustAttributes() 27 | if err != nil { 28 | return nil, diags.Append(&hcl.Diagnostic{ 29 | Severity: hcl.DiagError, 30 | Summary: "Failed to decode locals", 31 | Detail: err.Error(), 32 | Subject: l.Body.MissingItemRange().Ptr(), 33 | }) 34 | } 35 | 36 | for attr, expr := range attr { 37 | locals = append(locals, &Local{ 38 | Key: attr, 39 | Value: expr.Expr, 40 | }) 41 | } 42 | return locals, diags 43 | } 44 | 45 | func (l *Local) Description() Description { 46 | return Description{Name: l.Key} 47 | } 48 | 49 | func (l *Local) Identifier() string { 50 | return fmt.Sprintf("local.%s", l.Key) 51 | } 52 | 53 | func (*Local) Type() string { 54 | return LocalBlock 55 | } 56 | 57 | func (*Local) IsDaemon() bool { 58 | return false 59 | } 60 | 61 | func (*Local) Terminate(conductor *Conductor, safe bool) hcl.Diagnostics { 62 | return nil 63 | } 64 | 65 | func (*Local) Kill() hcl.Diagnostics { 66 | return nil 67 | } 68 | 69 | func (*Local) Set(k any, v any) { 70 | } 71 | 72 | func (*Local) Get(k any) any { 73 | return nil 74 | } 75 | 76 | func (l LocalsGroup) Expand() ([]*Local, hcl.Diagnostics) { 77 | var diags hcl.Diagnostics 78 | var locals []*Local 79 | for _, lo := range l { 80 | ll, dd := lo.Expand() 81 | diags = diags.Extend(dd) 82 | locals = append(locals, ll...) 83 | } 84 | return locals, diags 85 | } 86 | 87 | func (l LocalGroup) ById(id string) (*Local, hcl.Diagnostics) { 88 | for _, lo := range l { 89 | if lo.Key == id { 90 | return lo, nil 91 | } 92 | } 93 | return nil, hcl.Diagnostics{ 94 | { 95 | Severity: hcl.DiagError, 96 | Summary: "Local not found", 97 | Detail: "Local with id " + id + " not found", 98 | }, 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /internal/ci/locals_hcl.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | func (l *Local) Variables() []hcl.Traversal { 6 | return l.Value.Variables() 7 | } 8 | -------------------------------------------------------------------------------- /internal/ci/locals_lifecycle.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "github.com/hashicorp/hcl/v2" 6 | ) 7 | 8 | func (l *Local) ExecutionOptions(ctx context.Context) (*DaemonLifecycleConfig, hcl.Diagnostics) { 9 | return nil, nil 10 | } 11 | -------------------------------------------------------------------------------- /internal/ci/locals_prop.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | func (l *Local) Override() bool { 6 | return false 7 | } 8 | 9 | // CheckIfDistinct checks if the locals in l and ll are distinct 10 | func (l LocalGroup) CheckIfDistinct(ll LocalGroup) hcl.Diagnostics { 11 | var diags hcl.Diagnostics 12 | for _, local := range l { 13 | for _, local2 := range ll { 14 | if local.Identifier() == local2.Identifier() { 15 | diags = append(diags, &hcl.Diagnostic{ 16 | Severity: hcl.DiagError, 17 | Summary: "Duplicate local", 18 | Detail: "Local with id " + local.Identifier() + " is defined more than once", 19 | }) 20 | } 21 | } 22 | } 23 | return diags 24 | } 25 | -------------------------------------------------------------------------------- /internal/ci/locals_prop_test.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestLocal_Override(t *testing.T) { 9 | assert.Equal(t, (&Local{}).Override(), false) 10 | } 11 | 12 | func TestLocals_CheckIfDistinct(t *testing.T) { 13 | local1 := LocalGroup{ 14 | &Local{ 15 | Key: "local1", 16 | }, 17 | &Local{ 18 | Key: "local2", 19 | }, 20 | } 21 | 22 | local2 := LocalGroup{ 23 | &Local{ 24 | Key: "local3", 25 | }, 26 | &Local{ 27 | Key: "local4", 28 | }, 29 | } 30 | 31 | assert.Equal(t, local1.CheckIfDistinct(local2).HasErrors(), false) 32 | assert.Equal(t, local1.CheckIfDistinct(local1).HasErrors(), true) 33 | } 34 | -------------------------------------------------------------------------------- /internal/ci/locals_retry.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | func (l *Local) CanRetry() bool { 4 | return false 5 | } 6 | 7 | func (l *Local) MaxRetries() int { 8 | return 0 9 | } 10 | 11 | func (l *Local) MinRetryBackoff() int { 12 | return 0 13 | } 14 | func (l *Local) MaxRetryBackoff() int { 15 | return 0 16 | } 17 | 18 | func (l *Local) RetryExponentialBackoff() bool { 19 | return false 20 | } 21 | -------------------------------------------------------------------------------- /internal/ci/locals_retry_test.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "testing" 4 | 5 | func TestLocal_CanRetry(t *testing.T) { 6 | local := Local{} 7 | if local.CanRetry() { 8 | t.Error("CanRetry() should return false") 9 | } 10 | } 11 | 12 | func TestLocal_Description(t *testing.T) { 13 | local := Local{} 14 | if local.Description().Description != "" { 15 | t.Error("Description() should return empty string") 16 | } 17 | } 18 | 19 | func TestLocal_MaxRetries(t *testing.T) { 20 | local := Local{} 21 | if local.MaxRetries() != 0 { 22 | t.Error("MaxRetries() should return 0") 23 | } 24 | } 25 | 26 | func TestLocal_MaxRetryBackoff(t *testing.T) { 27 | local := Local{} 28 | if local.MaxRetryBackoff() != 0 { 29 | t.Error("MaxRetryBackoff() should return 0") 30 | } 31 | } 32 | 33 | func TestLocal_MinRetryBackoff(t *testing.T) { 34 | local := Local{} 35 | if local.MinRetryBackoff() != 0 { 36 | t.Error("MinRetryBackoff() should return 0") 37 | } 38 | } 39 | 40 | func TestLocal_RetryExponentialBackoff(t *testing.T) { 41 | local := Local{} 42 | if local.RetryExponentialBackoff() { 43 | t.Error("RetryExponentialBackoff() should return false") 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /internal/ci/locals_run.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/srevinsaju/togomak/v1/internal/global" 6 | "github.com/srevinsaju/togomak/v1/internal/runnable" 7 | "github.com/zclconf/go-cty/cty" 8 | ) 9 | 10 | func (l *Local) Run(conductor *Conductor, options ...runnable.Option) (diags hcl.Diagnostics) { 11 | 12 | logger := conductor.Logger().WithField("local", l.Key) 13 | logger.Debugf("running %s.%s", LocalBlock, l.Key) 14 | evalContext := conductor.Eval().Context() 15 | 16 | // region: mutating the data map 17 | // TODO: move it to a dedicated helper function 18 | 19 | global.LocalBlockEvalContextMutex.Lock() 20 | 21 | conductor.Eval().Mutex().RLock() 22 | locals := evalContext.Variables[LocalBlock] 23 | var localMutated map[string]cty.Value 24 | if locals.IsNull() { 25 | localMutated = make(map[string]cty.Value) 26 | } else { 27 | localMutated = locals.AsValueMap() 28 | } 29 | v, d := l.Value.Value(evalContext) 30 | conductor.Eval().Mutex().RUnlock() 31 | 32 | diags = diags.Extend(d) 33 | localMutated[l.Key] = v 34 | 35 | conductor.Eval().Mutex().Lock() 36 | evalContext.Variables[LocalBlock] = cty.ObjectVal(localMutated) 37 | conductor.Eval().Mutex().Unlock() 38 | 39 | global.LocalBlockEvalContextMutex.Unlock() 40 | 41 | // endregion 42 | 43 | return diags 44 | } 45 | 46 | func (l *Local) CanRun(conductor *Conductor, options ...runnable.Option) (bool, hcl.Diagnostics) { 47 | return true, nil 48 | } 49 | 50 | func (l *Local) Prepare(conductor *Conductor, skip bool, overridden bool) hcl.Diagnostics { 51 | return nil 52 | } 53 | 54 | func (l *Local) Terminated() bool { 55 | return true 56 | } 57 | -------------------------------------------------------------------------------- /internal/ci/locals_schema.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/zclconf/go-cty/cty" 6 | ) 7 | 8 | type Locals struct { 9 | Body hcl.Body `hcl:",remain"` 10 | } 11 | 12 | type Local struct { 13 | Key string 14 | Value hcl.Expression `hcl:"value,optional"` 15 | value cty.Value 16 | } 17 | 18 | type LocalsGroup []Locals 19 | type LocalGroup []*Local 20 | -------------------------------------------------------------------------------- /internal/ci/locals_test.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestLocals_Description(t *testing.T) { 9 | locals := Locals{} 10 | if locals.Description().Description != "" { 11 | t.Error("Description() should return empty string") 12 | } 13 | } 14 | 15 | func TestLocal_Kill(t *testing.T) { 16 | local := Local{} 17 | if local.Kill() != nil { 18 | t.Error("Kill() should return nil") 19 | } 20 | } 21 | 22 | func TestLocal_Terminate(t *testing.T) { 23 | local := Local{} 24 | if local.Terminate(nil, false) != nil { 25 | t.Error("Terminate() should return nil") 26 | } 27 | } 28 | 29 | func TestLocal_IsDaemon(t *testing.T) { 30 | local := Local{} 31 | if local.IsDaemon() { 32 | t.Error("IsDaemon() should return false") 33 | } 34 | } 35 | 36 | func TestLocal_Identifier(t *testing.T) { 37 | local := Local{Key: "test"} 38 | if local.Identifier() != "local.test" { 39 | t.Error("Identifier() should return 'local.test'") 40 | } 41 | } 42 | 43 | func TestLocal_Type(t *testing.T) { 44 | local := Local{} 45 | if local.Type() != LocalBlock { 46 | t.Error("Type() should return 'local'") 47 | } 48 | } 49 | 50 | func TestLocal_Set(t *testing.T) { 51 | data := Local{} 52 | data.Set("key", "value") 53 | } 54 | func TestLocal_Get(t *testing.T) { 55 | data := Local{} 56 | assert.Equal(t, data.Get("key"), nil) 57 | } 58 | -------------------------------------------------------------------------------- /internal/ci/macro.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/srevinsaju/togomak/v1/internal/blocks" 6 | ) 7 | 8 | func (m *Macro) Description() Description { 9 | return Description{Name: m.Id} 10 | } 11 | 12 | func (m *Macro) Identifier() string { 13 | return m.Id 14 | } 15 | 16 | func (m Macros) ById(id string) (*Macro, hcl.Diagnostics) { 17 | for _, macro := range m { 18 | if macro.Id == id { 19 | return ¯o, nil 20 | } 21 | } 22 | return nil, hcl.Diagnostics{ 23 | { 24 | Severity: hcl.DiagError, 25 | Summary: "Macro not found", 26 | Detail: "Macro with id " + id + " not found", 27 | }, 28 | } 29 | } 30 | 31 | func (m *Macro) Type() string { 32 | return blocks.MacroBlock 33 | } 34 | 35 | func (m *Macro) Variables() []hcl.Traversal { 36 | var traversal []hcl.Traversal 37 | 38 | traversal = append(traversal, m.Files.Variables()...) 39 | if m.Stage != nil { 40 | traversal = append(traversal, m.Stage.Variables()...) 41 | 42 | } 43 | return traversal 44 | } 45 | 46 | func (m *Macro) IsDaemon() bool { 47 | return false 48 | } 49 | 50 | func (m *Macro) Terminate(conductor *Conductor, safe bool) hcl.Diagnostics { 51 | return nil 52 | } 53 | 54 | func (m *Macro) Kill() hcl.Diagnostics { 55 | return nil 56 | } 57 | 58 | func (m *Macro) Set(k any, v any) { 59 | } 60 | 61 | func (m *Macro) Get(k any) any { 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /internal/ci/macro_lifecycle.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "github.com/hashicorp/hcl/v2" 6 | ) 7 | 8 | func (m *Macro) ExecutionOptions(ctx context.Context) (*DaemonLifecycleConfig, hcl.Diagnostics) { 9 | return nil, nil 10 | } 11 | -------------------------------------------------------------------------------- /internal/ci/macro_logging.go: -------------------------------------------------------------------------------- 1 | package ci 2 | -------------------------------------------------------------------------------- /internal/ci/macro_prop.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | func (m *Macro) Override() bool { 6 | return false 7 | } 8 | 9 | // CheckIfDistinct checks if the macro in m and mm are distinct 10 | func (m Macros) CheckIfDistinct(mm Macros) hcl.Diagnostics { 11 | for _, macro := range m { 12 | for _, macro2 := range mm { 13 | if macro.Identifier() == macro2.Identifier() { 14 | return hcl.Diagnostics{ 15 | { 16 | Severity: hcl.DiagError, 17 | Summary: "Duplicate macro", 18 | Detail: "Macro with id " + macro.Id + " is defined more than once", 19 | }, 20 | } 21 | } 22 | } 23 | } 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /internal/ci/macro_prop_test.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestMacro_Override(t *testing.T) { 9 | assert.Equal(t, (&Macro{}).Override(), false) 10 | } 11 | 12 | func TestMacros_CheckIfDistinct(t *testing.T) { 13 | macro1 := Macros{ 14 | Macro{ 15 | Id: "macro1", 16 | }, 17 | Macro{ 18 | Id: "macro2", 19 | }, 20 | } 21 | macro2 := Macros{ 22 | Macro{ 23 | Id: "macro3", 24 | }, 25 | Macro{ 26 | Id: "macro4", 27 | }, 28 | } 29 | 30 | assert.Equal(t, macro1.CheckIfDistinct(macro2).HasErrors(), false) 31 | assert.Equal(t, macro1.CheckIfDistinct(macro1).HasErrors(), true) 32 | } 33 | -------------------------------------------------------------------------------- /internal/ci/macro_retry.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | func (m *Macro) CanRetry() bool { 4 | return false 5 | } 6 | 7 | func (m *Macro) MaxRetries() int { 8 | return 0 9 | } 10 | 11 | func (m *Macro) MinRetryBackoff() int { 12 | return 0 13 | } 14 | func (m *Macro) MaxRetryBackoff() int { 15 | return 0 16 | } 17 | 18 | func (m *Macro) RetryExponentialBackoff() bool { 19 | return false 20 | } 21 | -------------------------------------------------------------------------------- /internal/ci/macro_retry_test.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestMacro_CanRetry(t *testing.T) { 9 | macro := Macro{} 10 | if macro.CanRetry() { 11 | t.Error("CanRetry() should return false") 12 | } 13 | } 14 | 15 | func TestMacro_Description(t *testing.T) { 16 | macro := Macro{} 17 | if macro.Description().Description != "" { 18 | t.Error("Description() should return empty string") 19 | } 20 | } 21 | 22 | func TestMacro_MaxRetries(t *testing.T) { 23 | macro := Macro{} 24 | if macro.MaxRetries() != 0 { 25 | t.Error("MaxRetries() should return 0") 26 | } 27 | } 28 | 29 | func TestMacro_MaxRetryBackoff(t *testing.T) { 30 | macro := Macro{} 31 | if macro.MaxRetryBackoff() != 0 { 32 | t.Error("MaxRetryBackoff() should return 0") 33 | } 34 | } 35 | 36 | func TestMacro_MinRetryBackoff(t *testing.T) { 37 | macro := Macro{} 38 | if macro.MinRetryBackoff() != 0 { 39 | t.Error("MinRetryBackoff() should return 0") 40 | } 41 | } 42 | 43 | func TestMacro_RetryExponentialBackoff(t *testing.T) { 44 | macro := Macro{} 45 | if macro.RetryExponentialBackoff() { 46 | t.Error("RetryExponentialBackoff() should return false") 47 | } 48 | } 49 | 50 | func TestMacro_Set(t *testing.T) { 51 | data := Macro{} 52 | data.Set("key", "value") 53 | } 54 | 55 | func TestMacro_Get(t *testing.T) { 56 | data := Macro{} 57 | assert.Equal(t, data.Get("key"), nil) 58 | } 59 | -------------------------------------------------------------------------------- /internal/ci/macro_run.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/srevinsaju/togomak/v1/internal/blocks" 6 | "github.com/srevinsaju/togomak/v1/internal/global" 7 | "github.com/srevinsaju/togomak/v1/internal/runnable" 8 | "github.com/zclconf/go-cty/cty" 9 | ) 10 | 11 | const ( 12 | SourceTypeGit = "git" 13 | ) 14 | 15 | func (m *Macro) Prepare(conductor *Conductor, skip bool, overridden bool) hcl.Diagnostics { 16 | return nil // no-op 17 | } 18 | 19 | func (m *Macro) Run(conductor *Conductor, options ...runnable.Option) (diags hcl.Diagnostics) { 20 | // _ := ctx.Value(TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) 21 | logger := conductor.Logger().WithField("macro", m.Id) 22 | logger.Tracef("running %s.%s", blocks.MacroBlock, m.Id) 23 | evalContext := conductor.Eval().Context() 24 | 25 | // region: mutating the data map 26 | // TODO: move it to a dedicated helper function 27 | 28 | global.MacroBlockEvalContextMutex.Lock() 29 | 30 | conductor.Eval().Mutex().RLock() 31 | macro := evalContext.Variables[blocks.MacroBlock] 32 | 33 | var macroMutated map[string]cty.Value 34 | if macro.IsNull() { 35 | macroMutated = make(map[string]cty.Value) 36 | } else { 37 | macroMutated = macro.AsValueMap() 38 | } 39 | // -> update r.Value accordingly 40 | f, d := m.Files.Value(evalContext) 41 | conductor.Eval().Mutex().RUnlock() 42 | 43 | if d != nil { 44 | diags = diags.Extend(d) 45 | } 46 | macroMutated[m.Id] = cty.ObjectVal(map[string]cty.Value{ 47 | "files": f, 48 | }) 49 | 50 | conductor.Eval().Mutex().Lock() 51 | evalContext.Variables[blocks.MacroBlock] = cty.ObjectVal(macroMutated) 52 | conductor.Eval().Mutex().Unlock() 53 | 54 | global.MacroBlockEvalContextMutex.Unlock() 55 | // endregion 56 | 57 | return diags 58 | } 59 | 60 | func (m *Macro) CanRun(conductor *Conductor, options ...runnable.Option) (ok bool, diags hcl.Diagnostics) { 61 | return false, diags 62 | } 63 | 64 | func (m *Macro) Terminated() bool { 65 | return true 66 | } 67 | -------------------------------------------------------------------------------- /internal/ci/macro_schema.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | type macroSourceSpec struct { 6 | stages []string 7 | } 8 | 9 | type Macro struct { 10 | Id string `hcl:"id,label" json:"id"` 11 | 12 | Source string `hcl:"source,optional" json:"source"` 13 | Files hcl.Expression `hcl:"files,optional" json:"files"` 14 | 15 | Parameters []string `hcl:"parameters,optional" json:"parameters"` 16 | Stage *Stage `hcl:"stage,block" json:"stage"` 17 | 18 | sourceSpec *macroSourceSpec 19 | } 20 | 21 | type Macros []Macro 22 | -------------------------------------------------------------------------------- /internal/ci/module.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/srevinsaju/togomak/v1/internal/blocks" 6 | ) 7 | 8 | func (m *Module) Description() Description { 9 | return Description{ 10 | Name: m.Name, 11 | Description: "", 12 | } 13 | } 14 | 15 | func (m *Module) Identifier() string { 16 | if m.Id == "" { 17 | panic("id not set") 18 | } 19 | return m.Id 20 | } 21 | 22 | func (i Modules) ById(id string) (*Module, hcl.Diagnostics) { 23 | for _, macro := range i { 24 | if macro.Identifier() == id { 25 | return ¯o, nil 26 | } 27 | } 28 | return nil, hcl.Diagnostics{ 29 | { 30 | Severity: hcl.DiagError, 31 | Summary: "module not found", 32 | Detail: "module with id " + id + " not found", 33 | }, 34 | } 35 | } 36 | 37 | func (m *Module) Type() string { 38 | return blocks.ModuleBlock 39 | } 40 | 41 | func (m *Module) IsDaemon() bool { 42 | return false 43 | } 44 | 45 | func (m *Module) Terminate(conductor *Conductor, safe bool) hcl.Diagnostics { 46 | return nil 47 | } 48 | 49 | func (m *Module) Kill() hcl.Diagnostics { 50 | return nil 51 | } 52 | 53 | func (m *Module) Set(k any, v any) { 54 | } 55 | 56 | func (m *Module) Get(k any) any { 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /internal/ci/module_hcl.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | func (m *Module) Variables() []hcl.Traversal { 6 | var vars []hcl.Traversal 7 | vars = append(vars, m.Source.Variables()...) 8 | vars = append(vars, m.DependsOn.Variables()...) 9 | vars = append(vars, m.Condition.Variables()...) 10 | vars = append(vars, m.ForEach.Variables()...) 11 | return vars 12 | } 13 | -------------------------------------------------------------------------------- /internal/ci/module_lifecycle.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "github.com/hashicorp/hcl/v2" 6 | ) 7 | 8 | func (m *Module) ExecutionOptions(ctx context.Context) (*DaemonLifecycleConfig, hcl.Diagnostics) { 9 | if m.Daemon != nil { 10 | return m.Daemon.Lifecycle.Parse(ctx) 11 | } 12 | return nil, nil 13 | } 14 | 15 | func (m *Module) LifecycleConfig() *Lifecycle { 16 | return m.Lifecycle 17 | } 18 | -------------------------------------------------------------------------------- /internal/ci/module_logger.go: -------------------------------------------------------------------------------- 1 | package ci 2 | -------------------------------------------------------------------------------- /internal/ci/module_prop.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | func (m *Module) Override() bool { 6 | return false 7 | } 8 | 9 | // CheckIfDistinct checks if the macro in m and mm are distinct 10 | func (m Modules) CheckIfDistinct(mm Modules) hcl.Diagnostics { 11 | for _, macro := range m { 12 | for _, macro2 := range mm { 13 | if macro.Identifier() == macro2.Identifier() { 14 | return hcl.Diagnostics{ 15 | { 16 | Severity: hcl.DiagError, 17 | Summary: "Duplicate macro", 18 | Detail: "module with id " + macro.Id + " is defined more than once", 19 | }, 20 | } 21 | } 22 | } 23 | } 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /internal/ci/module_retry.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | func (m *Module) CanRetry() bool { 4 | if m.Retry != nil { 5 | return m.Retry.Enabled 6 | } 7 | return false 8 | } 9 | 10 | func (m *Module) MaxRetries() int { 11 | return m.Retry.Attempts 12 | } 13 | 14 | func (m *Module) MinRetryBackoff() int { 15 | return m.Retry.MinBackoff 16 | } 17 | func (m *Module) MaxRetryBackoff() int { 18 | return m.Retry.MaxBackoff 19 | } 20 | 21 | func (m *Module) RetryExponentialBackoff() bool { 22 | return m.Retry.ExponentialBackoff 23 | } 24 | -------------------------------------------------------------------------------- /internal/ci/module_schema.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | ) 6 | 7 | type Module struct { 8 | Id string `hcl:"id,label" json:"id"` 9 | Name string `hcl:"name,optional" json:"name"` 10 | 11 | DependsOn hcl.Expression `hcl:"depends_on,optional" json:"depends_on"` 12 | Condition hcl.Expression `hcl:"if,optional" json:"if"` 13 | ForEach hcl.Expression `hcl:"for_each,optional" json:"for_each"` 14 | 15 | Source hcl.Expression `hcl:"source" json:"source"` 16 | 17 | pipeline *Pipeline 18 | 19 | Lifecycle *Lifecycle `hcl:"lifecycle,block" json:"lifecycle"` 20 | Retry *StageRetry `hcl:"retry,block" json:"retry"` 21 | Daemon *StageDaemon `hcl:"daemon,block" json:"daemon"` 22 | 23 | Body hcl.Body `hcl:",remain" json:"body"` 24 | } 25 | 26 | type Modules []Module 27 | -------------------------------------------------------------------------------- /internal/ci/output.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | const OutputBlock = "output" 4 | -------------------------------------------------------------------------------- /internal/ci/pipeline.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/sirupsen/logrus" 6 | "github.com/srevinsaju/togomak/v1/internal/meta" 7 | ) 8 | 9 | const PipelineBlock = "pipeline" 10 | 11 | type Pipeline struct { 12 | Builder Builder `hcl:"togomak,block" json:"togomak"` 13 | 14 | Stages Stages `hcl:"stage,block" json:"stages"` 15 | Data Datas `hcl:"data,block" json:"data"` 16 | Vars Variables `hcl:"variable,block" json:"variables"` 17 | Macros Macros `hcl:"macro,block" json:"macro"` 18 | Locals LocalsGroup `hcl:"locals,block" json:"locals"` 19 | Imports Imports `hcl:"import,block" json:"import"` 20 | 21 | Modules Modules `hcl:"module,block" json:"modules"` 22 | 23 | DataProviders DataProviders `hcl:"provider,block" json:"providers"` 24 | 25 | // private stuff 26 | Local LocalGroup 27 | 28 | Pre *PreStage `hcl:"pre,block" json:"pre"` 29 | Post *PostStage `hcl:"post,block" json:"post"` 30 | } 31 | 32 | func (pipe *Pipeline) Variables() []hcl.Traversal { 33 | var traversal []hcl.Traversal 34 | traversal = append(traversal, pipe.Stages.Variables()...) 35 | traversal = append(traversal, pipe.Data.Variables()...) 36 | traversal = append(traversal, pipe.DataProviders.Variables()...) 37 | return traversal 38 | } 39 | 40 | func (pipe *Pipeline) Logger() *logrus.Entry { 41 | return logrus.WithField("pipeline", "") 42 | } 43 | 44 | func (pipe *Pipeline) Resolve(runnableId string) (Block, bool, hcl.Diagnostics) { 45 | var runnable Block 46 | var diags hcl.Diagnostics 47 | var d hcl.Diagnostics 48 | 49 | skip := false 50 | switch runnableId { 51 | case meta.RootStage: 52 | skip = true 53 | case meta.PreStage: 54 | if pipe.Pre == nil { 55 | pipe.Logger().Debugf("skipping runnable pre block %s, not defined", runnableId) 56 | skip = true 57 | break 58 | } 59 | runnable = pipe.Pre.ToStage() 60 | case meta.PostStage: 61 | if pipe.Post == nil { 62 | pipe.Logger().Debugf("skipping runnable post block %s, not defined", runnableId) 63 | skip = true 64 | break 65 | } 66 | runnable = pipe.Post.ToStage() 67 | default: 68 | runnable, d = Resolve(pipe, runnableId) 69 | diags = diags.Extend(d) 70 | } 71 | return runnable, skip, diags 72 | } 73 | -------------------------------------------------------------------------------- /internal/ci/pipeline_expand.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "path/filepath" 6 | ) 7 | 8 | func (pipe *Pipeline) ExpandImports(conductor *Conductor, pwd string) (*Pipeline, hcl.Diagnostics) { 9 | var pipes MetaList 10 | var diags hcl.Diagnostics 11 | pipes = pipes.Append(NewMeta(pipe, nil, "memory")) 12 | tmpDir := conductor.TempDir() 13 | 14 | dst, err := filepath.Abs(filepath.Join(tmpDir, "import")) 15 | if err != nil { 16 | panic(err) 17 | } 18 | m := pipe.Imports 19 | for _, im := range m { 20 | p, d := im.Expand(conductor, pwd, dst) 21 | diags = diags.Extend(d) 22 | if d.HasErrors() { 23 | continue 24 | } 25 | pipes = pipes.Append(NewMeta(p, nil, im.Identifier())) 26 | } 27 | p, d := Merge(pipes) 28 | diags = diags.Extend(d) 29 | return p, diags 30 | } 31 | -------------------------------------------------------------------------------- /internal/ci/pipeline_import.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/srevinsaju/togomak/v1/internal/path" 6 | ) 7 | 8 | func ExpandImports(conductor *Conductor, pipe *Pipeline, paths *path.Path) (*Pipeline, hcl.Diagnostics) { 9 | var d hcl.Diagnostics 10 | var diags hcl.Diagnostics 11 | 12 | if len(pipe.Imports) != 0 { 13 | pipe.Logger().Debugf("expanding imports") 14 | d = pipe.Imports.PopulateProperties(conductor) 15 | diags = diags.Extend(d) 16 | if d.HasErrors() { 17 | return pipe, diags 18 | } 19 | 20 | pipe.Logger().Debugf("populating properties for imports completed with %d error(s)", len(d.Errs())) 21 | 22 | pipe, d = pipe.ExpandImports(conductor, paths.Cwd) 23 | diags = diags.Extend(d) 24 | pipe.Logger().Debugf("expanding imports completed with %d error(s)", len(d.Errs())) 25 | 26 | } 27 | return pipe, diags 28 | } 29 | -------------------------------------------------------------------------------- /internal/ci/pipeline_outputs.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/go-envparse" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/srevinsaju/togomak/v1/internal/meta" 7 | "github.com/srevinsaju/togomak/v1/internal/x" 8 | "github.com/zclconf/go-cty/cty" 9 | "os" 10 | "path/filepath" 11 | ) 12 | 13 | func ExpandOutputs(conductor *Conductor) hcl.Diagnostics { 14 | var diags hcl.Diagnostics 15 | logger := conductor.Logger().WithField("orchestra", "outputs") 16 | togomakEnvFile := filepath.Join(conductor.Process.TempDir, meta.OutputEnvFile) 17 | logger.Tracef("%s will be stored and exported here: %s", meta.OutputEnvVar, togomakEnvFile) 18 | envFile, err := os.OpenFile(togomakEnvFile, os.O_RDONLY|os.O_CREATE, 0644) 19 | if err == nil { 20 | e, err := envparse.Parse(envFile) 21 | if err != nil { 22 | diags = diags.Append(&hcl.Diagnostic{ 23 | Severity: hcl.DiagError, 24 | Summary: "could not parse TOGOMAK_ENV file", 25 | Detail: err.Error(), 26 | }) 27 | return diags 28 | } 29 | x.Must(envFile.Close()) 30 | ee := make(map[string]cty.Value) 31 | for k, v := range e { 32 | ee[k] = cty.StringVal(v) 33 | } 34 | conductor.Eval().Mutex().Lock() 35 | conductor.Eval().Context().Variables[OutputBlock] = cty.ObjectVal(ee) 36 | conductor.Eval().Mutex().Unlock() 37 | } else { 38 | logger.Warnf("could not open %s file, ignoring... :%s", meta.OutputEnvVar, err) 39 | } 40 | return diags 41 | } 42 | -------------------------------------------------------------------------------- /internal/ci/pipeline_read.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/hashicorp/hcl/v2/gohcl" 7 | "github.com/srevinsaju/togomak/v1/internal/global" 8 | "github.com/srevinsaju/togomak/v1/internal/meta" 9 | "github.com/srevinsaju/togomak/v1/internal/parse" 10 | "github.com/srevinsaju/togomak/v1/internal/ui" 11 | "os" 12 | "path/filepath" 13 | "strings" 14 | ) 15 | 16 | // Read reads togomak.hcl from the configuration file directory. A configuration file directory is the one that 17 | // contains togomak.hcl, it searches recursively outwards. 18 | // DEPRECATED: use ReadDir instead 19 | func Read(conductor *Conductor) (*Pipeline, hcl.Diagnostics) { 20 | ciFile := parse.ConfigFilePath(conductor.Config.Paths) 21 | f, diags := conductor.Parser.ParseHCLFile(ciFile) 22 | 23 | if diags.HasErrors() { 24 | return nil, diags 25 | } 26 | 27 | pipeline := &Pipeline{} 28 | diags = gohcl.DecodeBody(f.Body, nil, pipeline) 29 | 30 | if pipeline.Builder.Version != 1 { 31 | return ReadDir(conductor) 32 | } else if pipeline.Builder.Version == 1 { 33 | ui.DeprecationWarning(fmt.Sprintf("%s configuration version 1 is deprecated, and support for the same will be removed in a later version. ", meta.AppName)) 34 | } 35 | return pipeline, diags 36 | } 37 | 38 | // ReadDir parses an entire directory of *.hcl files and merges them together. This is useful when you want to 39 | // split your pipeline into multiple files, without having to import them individually 40 | func ReadDir(conductor *Conductor) (*Pipeline, hcl.Diagnostics) { 41 | dir := parse.ConfigFileDir(conductor.Config.Paths) 42 | return ReadDirFromPath(conductor, dir) 43 | 44 | } 45 | 46 | func ReadDirFromPath(conductor *Conductor, dir string) (*Pipeline, hcl.Diagnostics) { 47 | logger := global.Logger() 48 | var diags hcl.Diagnostics 49 | togomakFiles, err := os.ReadDir(dir) 50 | if err != nil { 51 | panic(err) 52 | } 53 | var pipes []*Meta 54 | for _, file := range togomakFiles { 55 | if file.IsDir() { 56 | continue 57 | } 58 | if !strings.HasSuffix(file.Name(), ".hcl") { 59 | continue 60 | } 61 | if strings.Contains(file.Name(), ".lock.hcl") { 62 | // we will not process .lock.hcl files 63 | continue 64 | } 65 | 66 | f, d := conductor.Parser.ParseHCLFile(filepath.Join(dir, file.Name())) 67 | diags = diags.Extend(d) 68 | 69 | p := &Pipeline{} 70 | 71 | d = gohcl.DecodeBody(f.Body, nil, p) 72 | diags = diags.Extend(d) 73 | if d.HasErrors() { 74 | logger.Debugf("error parsing %s", file.Name()) 75 | continue 76 | } 77 | pipes = append(pipes, &Meta{ 78 | pipe: p, 79 | f: f, 80 | filename: file.Name(), 81 | }) 82 | 83 | } 84 | pipe, d := Merge(pipes) 85 | diags = diags.Extend(d) 86 | return pipe, diags 87 | } 88 | -------------------------------------------------------------------------------- /internal/ci/pipeline_run.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/srevinsaju/togomak/v1/internal/c" 7 | "github.com/srevinsaju/togomak/v1/internal/dg" 8 | "github.com/srevinsaju/togomak/v1/internal/runnable" 9 | ) 10 | 11 | func StartHandlers(conductor *Conductor) *Handler { 12 | 13 | h := NewHandler( 14 | WithContext(conductor.Context()), 15 | WithLogger(conductor.RootLogger), 16 | WithDiagnosticWriter(conductor.DiagWriter), 17 | WithProcessBootTime(conductor.Process.BootTime), 18 | ) 19 | go h.Interrupt() 20 | go h.Kill() 21 | go h.Daemons() 22 | return h 23 | } 24 | 25 | func (pipe *Pipeline) Run(conductor *Conductor) (*Handler, dg.AbstractDiagnostics) { 26 | var d hcl.Diagnostics 27 | logger := conductor.Logger().WithField("orchestra", "run") 28 | cfg := conductor.Config 29 | ctx, cancel := context.WithCancel(conductor.Context()) 30 | h := StartHandlers(conductor) 31 | 32 | defer cancel() 33 | defer h.WriteDiagnostics() 34 | 35 | // --> expand imports 36 | pipe, d = ExpandImports(conductor, pipe, conductor.Config.Paths) 37 | h.Diags.Extend(d) 38 | if h.Diags.HasErrors() { 39 | return h, h.Diags 40 | } 41 | 42 | /// we will first expand all local blocks 43 | logger.Debugf("expanding local blocks") 44 | locals, d := pipe.Locals.Expand() 45 | h.Diags.Extend(d) 46 | if d.HasErrors() { 47 | return h, h.Diags 48 | } 49 | pipe.Local = locals 50 | 51 | // store the pipe in the context 52 | ctx = context.WithValue(ctx, c.TogomakContextPipeline, pipe) 53 | h = h.Update(WithContext(ctx)) 54 | conductor.Update(ConductorWithContext(ctx)) 55 | 56 | // --> generate a dependency graph 57 | // we will now generate a dependency graph from the pipeline 58 | // this will be used to generate the pipeline 59 | logger.Debugf("generating dependency graph") 60 | depGraph, d := GraphTopoSort(conductor, pipe) 61 | h.Diags.Extend(d) 62 | if h.Diags.HasErrors() { 63 | return h, h.Diags 64 | } 65 | 66 | // endregion: interrupt h 67 | opts := []runnable.Option{ 68 | runnable.WithBehavior(conductor.Config.Behavior), 69 | runnable.WithPaths(conductor.Config.Paths), 70 | } 71 | 72 | logger.Debugf("starting runnables") 73 | for _, layer := range depGraph.TopoSortedLayers() { 74 | // we parse the TOGOMAK_ENV file at the beginning of every layer 75 | // this allows us to have different environments for different layers 76 | 77 | d = ExpandOutputs(conductor) 78 | h.Diags.Extend(d) 79 | if h.Diags.HasErrors() { 80 | break 81 | } 82 | 83 | for _, runnableId := range layer { 84 | 85 | runnable, skip, d := pipe.Resolve(runnableId) 86 | if skip { 87 | continue 88 | } 89 | if d.HasErrors() { 90 | h.Diags.Extend(d) 91 | break 92 | } 93 | 94 | ok, overridden, d := BlockCanRun(runnable, conductor, runnableId, depGraph, opts...) 95 | h.Diags.Extend(d) 96 | if d.HasErrors() { 97 | break 98 | } 99 | 100 | // prepare step needs to pipeline.Run before the runnable is pipeline.Run 101 | // we will also need to prompt the user with the information saying that it has been skipped 102 | d = runnable.Prepare(conductor, !ok, overridden) 103 | h.Diags.Extend(d) 104 | if d.HasErrors() { 105 | break 106 | } 107 | 108 | if !ok { 109 | logger.Debugf("skipping runnable %s, condition evaluated to false", runnableId) 110 | continue 111 | } 112 | 113 | logger.Debugf("runnable %s is %T", runnableId, runnable) 114 | 115 | if runnable.IsDaemon() { 116 | h.Tracker.AppendDaemon(runnable) 117 | } else { 118 | h.Tracker.AppendRunnable(runnable) 119 | } 120 | 121 | go BlockRunWithRetries(conductor, runnableId, runnable, h, conductor.Logger(), opts...) 122 | 123 | if cfg.Pipeline.DryRun { 124 | // TODO: implement --concurrency option 125 | // wait for the runnable to finish 126 | // disable concurrency 127 | h.Tracker.RunnableWait() 128 | h.Tracker.DaemonWait() 129 | } 130 | if pipe.Builder.Behavior != nil && pipe.Builder.Behavior.DisableConcurrency { 131 | h.Tracker.RunnableWait() 132 | h.Tracker.DaemonWait() 133 | } 134 | } 135 | h.Tracker.RunnableWait() 136 | 137 | if h.Diags.HasErrors() { 138 | if h.Tracker.HasDaemons() && !cfg.Pipeline.DryRun && !cfg.Behavior.Unattended { 139 | logger.Info("pipeline failed, waiting for daemons to shut down") 140 | logger.Info("hit Ctrl+C to force stop them") 141 | // wait for daemons to stop 142 | h.Tracker.DaemonWait() 143 | } else if h.Tracker.HasDaemons() && !cfg.Pipeline.DryRun { 144 | logger.Info("pipeline failed, waiting for daemons to shut down...") 145 | // wait for daemons to stop 146 | cancel() 147 | } 148 | break 149 | } 150 | } 151 | 152 | h.Tracker.DaemonWait() 153 | return h, h.Diags 154 | } 155 | -------------------------------------------------------------------------------- /internal/ci/prop.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | type Overrideable interface { 4 | Override() bool 5 | } 6 | 7 | type Distinct interface { 8 | Overrideable 9 | Describable 10 | } 11 | -------------------------------------------------------------------------------- /internal/ci/query_model.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/hashicorp/hcl/v2/hclsyntax" 6 | "github.com/sirupsen/logrus" 7 | "github.com/srevinsaju/togomak/v1/internal/blocks" 8 | "github.com/srevinsaju/togomak/v1/internal/global" 9 | "github.com/zclconf/go-cty/cty" 10 | "strings" 11 | ) 12 | 13 | type QueryEngine struct { 14 | Rule hcl.Expression 15 | rule string 16 | Logger *logrus.Entry 17 | 18 | empty bool 19 | } 20 | 21 | func New(comp string) (*QueryEngine, hcl.Diagnostics) { 22 | var diags hcl.Diagnostics 23 | 24 | logger := global.Logger().WithField("rules", "") 25 | logger.Debugf("received rule '%s'", comp) 26 | if comp == "" || strings.Trim(comp, " ") == "" { 27 | return &QueryEngine{ 28 | empty: true, 29 | rule: comp, 30 | Logger: logger, 31 | }, diags 32 | } 33 | 34 | p, d := hclsyntax.ParseExpression([]byte(comp), "", hcl.Pos{Line: 1, Column: 1}) 35 | diags = diags.Extend(d) 36 | 37 | return &QueryEngine{ 38 | Rule: p, 39 | rule: comp, 40 | Logger: logger, 41 | }, diags 42 | } 43 | 44 | func (e *QueryEngine) Eval(conductor *Conductor, ok bool, stage PhasedBlock) (bool, bool, hcl.Diagnostics) { 45 | var diags hcl.Diagnostics 46 | var d hcl.Diagnostics 47 | if e.empty { 48 | return ok, false, diags 49 | } 50 | 51 | ectx := conductor.Eval().Context() 52 | ectx = ectx.NewChild() 53 | ectx.Variables = map[string]cty.Value{} 54 | ectx.Variables["if"] = cty.BoolVal(ok) 55 | ectx.Variables["id"] = cty.StringVal(stage.Identifier()) 56 | ectx.Variables["name"] = cty.StringVal(stage.Description().Name) 57 | 58 | lifecyclePhase := cty.ListVal([]cty.Value{cty.StringVal("default")}) 59 | lifecycleTimeout := cty.NumberIntVal(0) 60 | if stage.LifecycleConfig() != nil { 61 | conductor.Eval().Mutex().RLock() 62 | lifecyclePhase, d = stage.LifecycleConfig().Phase.Value(ectx) 63 | conductor.Eval().Mutex().RUnlock() 64 | 65 | diags = diags.Extend(d) 66 | 67 | conductor.Eval().Mutex().RLock() 68 | lifecycleTimeout, d = stage.LifecycleConfig().Timeout.Value(ectx) 69 | conductor.Eval().Mutex().RUnlock() 70 | diags = diags.Extend(d) 71 | } 72 | 73 | ectx.Variables[blocks.LifecycleBlock] = cty.ObjectVal(map[string]cty.Value{ 74 | "phase": lifecyclePhase, 75 | "timeout": lifecycleTimeout, 76 | }) 77 | 78 | conductor.Eval().Mutex().RLock() 79 | v, d := e.Rule.Value(ectx) 80 | conductor.Eval().Mutex().RUnlock() 81 | diags = diags.Extend(d) 82 | 83 | if v.Type() != cty.Bool { 84 | diags = diags.Append(&hcl.Diagnostic{ 85 | Severity: hcl.DiagError, 86 | Summary: "Rule must be a boolean expression", 87 | Detail: "The rule must be a boolean expression, but the given expression has type " + v.Type().FriendlyName(), 88 | }) 89 | return false, true, diags 90 | } 91 | if !v.IsKnown() || !v.IsWhollyKnown() { 92 | diags = diags.Append(&hcl.Diagnostic{ 93 | Severity: hcl.DiagError, 94 | Summary: "Rule must be a known value", 95 | Detail: "The rule must be a known value, but the given expression has unknown parts", 96 | }) 97 | } 98 | 99 | e.Logger.WithField("runnable", stage.Identifier()).Debugf("evaluated rule '%s' to %v", e.rule, v.True()) 100 | return v.True(), true, diags 101 | } 102 | -------------------------------------------------------------------------------- /internal/ci/query_slice.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | ) 6 | 7 | type QueryEngines []*QueryEngine 8 | 9 | func NewSlice(queries []string) (QueryEngines, hcl.Diagnostics) { 10 | var diags hcl.Diagnostics 11 | var engines QueryEngines 12 | for _, query := range queries { 13 | e, d := New(query) 14 | diags = diags.Extend(d) 15 | engines = append(engines, e) 16 | } 17 | return engines, diags 18 | } 19 | 20 | func (e QueryEngines) Eval(conductor *Conductor, ok bool, stage PhasedBlock) (bool, bool, hcl.Diagnostics) { 21 | var diags hcl.Diagnostics 22 | var d hcl.Diagnostics 23 | 24 | var overridden bool 25 | var resultOk bool 26 | 27 | for _, engine := range e { 28 | resultOk, overridden, d = engine.Eval(conductor, ok, stage) 29 | diags = diags.Extend(d) 30 | if d.HasErrors() { 31 | continue 32 | } 33 | if resultOk { 34 | return resultOk, overridden, diags 35 | } 36 | } 37 | return resultOk, overridden, diags 38 | } 39 | -------------------------------------------------------------------------------- /internal/ci/stage.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/hashicorp/hcl/v2" 7 | "github.com/srevinsaju/togomak/v1/internal/blocks" 8 | ) 9 | 10 | const StageContextChildStatuses = "child_statuses" 11 | 12 | func (s *Stage) Description() Description { 13 | return Description{ 14 | Name: s.Name, 15 | Description: "", 16 | } 17 | } 18 | 19 | func (s *Stage) Identifier() string { 20 | return s.Id 21 | } 22 | 23 | func (s *Stage) Set(k any, v any) { 24 | if s.ctxInitialised == false { 25 | s.ctx = context.Background() 26 | s.ctxInitialised = true 27 | } 28 | s.ctx = context.WithValue(s.ctx, k, v) 29 | } 30 | 31 | func (s *Stage) Get(k any) any { 32 | if s.ctxInitialised { 33 | return s.ctx.Value(k) 34 | } 35 | return nil 36 | } 37 | 38 | func (s *Stage) Type() string { 39 | return blocks.StageBlock 40 | } 41 | 42 | func (s *Stage) IsDaemon() bool { 43 | return s.Daemon != nil && s.Daemon.Enabled 44 | } 45 | 46 | func (s Stages) ById(id string) (*Stage, hcl.Diagnostics) { 47 | for _, stage := range s { 48 | if stage.Id == id { 49 | return &stage, nil 50 | } 51 | } 52 | return nil, hcl.Diagnostics{ 53 | { 54 | Severity: hcl.DiagError, 55 | Summary: "Stage not found", 56 | Detail: fmt.Sprintf("Stage with id %s not found", id), 57 | }, 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /internal/ci/stage_containers_hcl.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/docker/go-connections/nat" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/zclconf/go-cty/cty" 7 | ) 8 | 9 | // Nat returns a map of exposed ports and a map of port bindings 10 | // after parsing the HCL configuration from StageContainerPorts. 11 | func (s StageContainerPorts) Nat(conductor *Conductor, evalCtx *hcl.EvalContext) (map[nat.Port]struct{}, map[nat.Port][]nat.PortBinding, hcl.Diagnostics) { 12 | var hclDiags hcl.Diagnostics 13 | var rawPortSpecs []string 14 | for _, port := range s { 15 | conductor.Eval().Mutex().RLock() 16 | p, d := port.Port.Value(evalCtx) 17 | conductor.Eval().Mutex().RLock() 18 | 19 | hclDiags = hclDiags.Extend(d) 20 | if d.HasErrors() { 21 | continue 22 | } 23 | if p.Type() != cty.String { 24 | hclDiags = hclDiags.Append(&hcl.Diagnostic{ 25 | Severity: hcl.DiagError, 26 | Summary: "Invalid port specification", 27 | Detail: "Port specification must be a string", 28 | Subject: port.Port.Range().Ptr(), 29 | }) 30 | continue 31 | } 32 | 33 | rawPortSpecs = append(rawPortSpecs, p.AsString()) 34 | } 35 | 36 | exposedPorts, bindings, err := nat.ParsePortSpecs(rawPortSpecs) 37 | if err != nil { 38 | hclDiags = hclDiags.Append(&hcl.Diagnostic{ 39 | Severity: hcl.DiagError, 40 | Summary: "Invalid port specification", 41 | Detail: err.Error(), 42 | }) 43 | } 44 | return exposedPorts, bindings, hclDiags 45 | } 46 | -------------------------------------------------------------------------------- /internal/ci/stage_hcl.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | ) 6 | 7 | func (e *StageEnvironment) Variables() []hcl.Traversal { 8 | var traversal []hcl.Traversal 9 | traversal = append(traversal, e.Value.Variables()...) 10 | return traversal 11 | } 12 | func (e *StageContainerVolume) Variables() []hcl.Traversal { 13 | var traversal []hcl.Traversal 14 | traversal = append(traversal, e.Source.Variables()...) 15 | traversal = append(traversal, e.Destination.Variables()...) 16 | return traversal 17 | } 18 | 19 | func (e *StageDaemon) Variables() []hcl.Traversal { 20 | var traversal []hcl.Traversal 21 | if e.Lifecycle != nil { 22 | traversal = append(traversal, e.Lifecycle.Variables()...) 23 | } 24 | return traversal 25 | } 26 | 27 | func (e *StageContainerVolumes) Variables() []hcl.Traversal { 28 | var traversal []hcl.Traversal 29 | for _, volume := range *e { 30 | traversal = append(traversal, volume.Variables()...) 31 | } 32 | return traversal 33 | } 34 | 35 | func (s *CoreStage) Variables() []hcl.Traversal { 36 | var traversal []hcl.Traversal 37 | traversal = append(traversal, s.Condition.Variables()...) 38 | traversal = append(traversal, s.Dir.Variables()...) 39 | traversal = append(traversal, s.DependsOn.Variables()...) 40 | traversal = append(traversal, s.Script.Variables()...) 41 | traversal = append(traversal, s.Args.Variables()...) 42 | 43 | traversal = append(traversal, s.dependsOnVariablesMacro...) 44 | 45 | if s.Use != nil { 46 | traversal = append(traversal, s.Use.Macro.Variables()...) 47 | traversal = append(traversal, s.Use.Parameters.Variables()...) 48 | } 49 | if s.Container != nil { 50 | traversal = append(traversal, s.Container.Volumes.Variables()...) 51 | } 52 | if s.Daemon != nil { 53 | traversal = append(traversal, s.Daemon.Variables()...) 54 | } 55 | 56 | for _, env := range s.Environment { 57 | traversal = append(traversal, env.Variables()...) 58 | } 59 | if s.PostHook != nil { 60 | for _, hook := range s.PostHook { 61 | traversal = append(traversal, hook.Stage.Variables()...) 62 | } 63 | } 64 | if s.PreHook != nil { 65 | for _, hook := range s.PreHook { 66 | traversal = append(traversal, hook.Stage.Variables()...) 67 | } 68 | } 69 | return traversal 70 | } 71 | 72 | func (s *Stage) Variables() []hcl.Traversal { 73 | var traversal []hcl.Traversal 74 | traversal = append(traversal, s.CoreStage.Variables()...) 75 | if s.Lifecycle != nil { 76 | traversal = append(traversal, s.Lifecycle.Timeout.Variables()...) 77 | } 78 | return traversal 79 | } 80 | 81 | func (s Stages) Variables() []hcl.Traversal { 82 | var traversal []hcl.Traversal 83 | for _, stage := range s { 84 | traversal = append(traversal, stage.Variables()...) 85 | } 86 | return traversal 87 | } 88 | -------------------------------------------------------------------------------- /internal/ci/stage_hooks.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/srevinsaju/togomak/v1/internal/runnable" 7 | ) 8 | 9 | func (s *Stage) BeforeRun(conductor *Conductor, opts ...runnable.Option) hcl.Diagnostics { 10 | logger := conductor.Logger().WithField("stage", s.Id) 11 | if s.PreHook == nil { 12 | logger.Debug("no pre-hook defined") 13 | return nil 14 | } 15 | var diags hcl.Diagnostics 16 | 17 | for _, hook := range s.PreHook { 18 | diags = diags.Extend( 19 | (&Stage{fmt.Sprintf("%s.pre", s.Id), nil, hook.Stage, nil}).Run(conductor, opts...), 20 | ) 21 | } 22 | return diags 23 | } 24 | 25 | func (s *Stage) AfterRun(conductor *Conductor, opts ...runnable.Option) hcl.Diagnostics { 26 | logger := conductor.Logger().WithField("stage", s.Id) 27 | if s.PostHook == nil { 28 | logger.Debug("no post-hook defined") 29 | return nil 30 | } 31 | var diags hcl.Diagnostics 32 | 33 | for _, hook := range s.PostHook { 34 | diags = diags.Extend( 35 | (&Stage{fmt.Sprintf("%s.post", s.Id), nil, hook.Stage, nil}).Run(conductor, opts...), 36 | ) 37 | } 38 | return diags 39 | } 40 | -------------------------------------------------------------------------------- /internal/ci/stage_lifecycle.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "github.com/hashicorp/hcl/v2" 6 | ) 7 | 8 | func (s *Stage) ExecutionOptions(ctx context.Context) (*DaemonLifecycleConfig, hcl.Diagnostics) { 9 | if s.Daemon != nil { 10 | return s.Daemon.Lifecycle.Parse(ctx) 11 | } 12 | return nil, nil 13 | } 14 | 15 | func (s *Stage) LifecycleConfig() *Lifecycle { 16 | return s.Lifecycle 17 | } 18 | -------------------------------------------------------------------------------- /internal/ci/stage_logging.go: -------------------------------------------------------------------------------- 1 | package ci 2 | -------------------------------------------------------------------------------- /internal/ci/stage_prop.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/srevinsaju/togomak/v1/internal/blocks" 6 | "github.com/srevinsaju/togomak/v1/internal/x" 7 | ) 8 | 9 | func (s *Stage) Override() bool { 10 | return false 11 | } 12 | 13 | func (s Stages) Override() bool { 14 | return false 15 | } 16 | 17 | // CheckIfDistinct checks if the stages in s and ss are distinct 18 | // TODO: check if this is a good way to do this 19 | func (s Stages) CheckIfDistinct(ss Stages) hcl.Diagnostics { 20 | var diags hcl.Diagnostics 21 | for _, stage := range s { 22 | for _, stage2 := range ss { 23 | if stage.Id == stage2.Id { 24 | diags = append(diags, &hcl.Diagnostic{ 25 | Severity: hcl.DiagError, 26 | Summary: "Duplicate stage", 27 | Detail: "Stage with id " + stage.Id + " is defined more than once", 28 | }) 29 | } 30 | } 31 | } 32 | return diags 33 | } 34 | 35 | func (s *Stage) String() string { 36 | return x.RenderBlock(blocks.StageBlock, s.Id) 37 | } 38 | -------------------------------------------------------------------------------- /internal/ci/stage_prop_test.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestStage_Override(t *testing.T) { 9 | assert.Equal(t, (&Stage{}).Override(), false) 10 | } 11 | 12 | func TestStages_CheckIfDistinct(t *testing.T) { 13 | stage1 := Stages{ 14 | Stage{ 15 | Id: "stage1", 16 | }, 17 | Stage{ 18 | Id: "stage2", 19 | }, 20 | } 21 | stage2 := Stages{ 22 | Stage{ 23 | Id: "stage3", 24 | }, 25 | Stage{ 26 | Id: "stage4", 27 | }, 28 | } 29 | 30 | assert.Equal(t, stage1.CheckIfDistinct(stage2).HasErrors(), false) 31 | assert.Equal(t, stage1.CheckIfDistinct(stage1).HasErrors(), true) 32 | } 33 | 34 | func TestStages_Override(t *testing.T) { 35 | assert.Equal(t, (&Stages{}).Override(), false) 36 | } 37 | -------------------------------------------------------------------------------- /internal/ci/stage_retry.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | func (s *Stage) CanRetry() bool { 4 | if s.Retry != nil { 5 | return s.Retry.Enabled 6 | } 7 | return false 8 | } 9 | 10 | func (s *Stage) MaxRetries() int { 11 | return s.Retry.Attempts 12 | } 13 | 14 | func (s *Stage) MinRetryBackoff() int { 15 | return s.Retry.MinBackoff 16 | } 17 | func (s *Stage) MaxRetryBackoff() int { 18 | return s.Retry.MaxBackoff 19 | } 20 | 21 | func (s *Stage) RetryExponentialBackoff() bool { 22 | return s.Retry.ExponentialBackoff 23 | 24 | } 25 | -------------------------------------------------------------------------------- /internal/ci/stage_terminate.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/docker/docker/api/types" 7 | dockerContainer "github.com/docker/docker/api/types/container" 8 | dockerClient "github.com/docker/docker/client" 9 | "github.com/hashicorp/hcl/v2" 10 | "github.com/srevinsaju/togomak/v1/internal/runnable" 11 | "syscall" 12 | ) 13 | 14 | func (s *Stage) Terminate(conductor *Conductor, safe bool) hcl.Diagnostics { 15 | logger := conductor.Logger().WithField("stage", s.Id) 16 | logger.Debug("terminating stage") 17 | ctx := context.Background() 18 | var diags hcl.Diagnostics 19 | if safe { 20 | s.terminated = true 21 | } 22 | 23 | defer func() { 24 | diags = diags.Extend(s.AfterRun( 25 | conductor, 26 | runnable.WithHook(), 27 | runnable.WithStatus(runnable.StatusTerminated), 28 | runnable.WithParent(runnable.ParentConfig{Name: s.Name, Id: s.Id}), 29 | )) 30 | }() 31 | 32 | if s.Container != nil && s.ContainerId != "" { 33 | 34 | cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv, dockerClient.WithAPIVersionNegotiation()) 35 | if err != nil { 36 | diags = diags.Append(&hcl.Diagnostic{ 37 | Severity: hcl.DiagError, 38 | Summary: "failed to create docker client", 39 | Detail: fmt.Sprintf("%s: %s", dockerContainerSourceFmt(s.ContainerId), err.Error()), 40 | }) 41 | } 42 | logger.Debug("stopping container") 43 | err = cli.ContainerStop(ctx, s.ContainerId, dockerContainer.StopOptions{}) 44 | if err != nil { 45 | diags = diags.Append(&hcl.Diagnostic{ 46 | Severity: hcl.DiagError, 47 | Summary: "failed to stop container", 48 | Detail: fmt.Sprintf("%s: %s", dockerContainerSourceFmt(s.ContainerId), err.Error()), 49 | }) 50 | } 51 | logger.Debug("removing container") 52 | err = cli.ContainerRemove(ctx, s.ContainerId, types.ContainerRemoveOptions{ 53 | RemoveVolumes: true, 54 | }) 55 | logger.Debug("removed container") 56 | 57 | } else if s.process != nil && s.process.Process != nil { 58 | if s.process.ProcessState != nil { 59 | if s.process.ProcessState.Exited() { 60 | return diags 61 | } 62 | } 63 | err := s.process.Process.Signal(syscall.SIGTERM) 64 | if err != nil { 65 | diags = diags.Append(&hcl.Diagnostic{ 66 | Severity: hcl.DiagError, 67 | Summary: "failed to terminate process", 68 | Detail: err.Error(), 69 | }) 70 | } 71 | } 72 | logger.Debug("terminated stage") 73 | 74 | return diags 75 | } 76 | 77 | func (s *Stage) Kill() hcl.Diagnostics { 78 | diags := s.Terminate(nil, false) 79 | if s.process != nil && !s.process.ProcessState.Exited() { 80 | err := s.process.Process.Kill() 81 | if err != nil { 82 | diags = diags.Append(&hcl.Diagnostic{ 83 | Severity: hcl.DiagError, 84 | Summary: "couldn't kill stage", 85 | Detail: err.Error(), 86 | }) 87 | } 88 | } 89 | return diags 90 | } 91 | 92 | func (s *Stage) Terminated() bool { 93 | return s.terminated 94 | } 95 | -------------------------------------------------------------------------------- /internal/ci/stage_test.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestStage_Description(t *testing.T) { 9 | stage := Stage{} 10 | assert.Equal(t, stage.Description().Description, "") 11 | } 12 | 13 | func TestStage_Set(t *testing.T) { 14 | stage := Stage{} 15 | stage.Set("key", "value") 16 | assert.Equal(t, stage.Get("key"), "value") 17 | } 18 | 19 | func TestStage_Get(t *testing.T) { 20 | stage := Stage{} 21 | assert.Equal(t, stage.Get("key"), nil) 22 | } 23 | -------------------------------------------------------------------------------- /internal/ci/variable.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/srevinsaju/togomak/v1/internal/blocks" 7 | ) 8 | 9 | func (v *Variable) Name() string { 10 | return v.Id 11 | } 12 | 13 | func (v *Variable) Description() Description { 14 | return Description{ 15 | Name: v.Id, 16 | Description: v.Desc, 17 | } 18 | } 19 | 20 | func (v *Variable) Identifier() string { 21 | return v.Id 22 | } 23 | 24 | func (v *Variable) Set(k any, value any) { 25 | return // do nothing 26 | } 27 | 28 | func (v *Variable) Get(k any) any { 29 | return nil 30 | } 31 | 32 | func (v *Variable) Type() string { 33 | return blocks.VariableBlock 34 | } 35 | 36 | func (v *Variable) IsDaemon() bool { 37 | return false 38 | } 39 | 40 | func (s Variables) ById(id string) (*Variable, hcl.Diagnostics) { 41 | for _, variable := range s { 42 | if variable.Id == id { 43 | return variable, nil 44 | } 45 | } 46 | return nil, hcl.Diagnostics{ 47 | { 48 | Severity: hcl.DiagError, 49 | Summary: "Variable not found", 50 | Detail: fmt.Sprintf("variable input with id %s not found", id), 51 | }, 52 | } 53 | } 54 | 55 | // CheckIfDistinct checks if the stages in s and ss are distinct 56 | // TODO: check if this is a good way to do this 57 | func (s Variables) CheckIfDistinct(ss Variables) hcl.Diagnostics { 58 | var diags hcl.Diagnostics 59 | for _, stage := range s { 60 | for _, stage2 := range ss { 61 | if stage.Id == stage2.Id { 62 | diags = append(diags, &hcl.Diagnostic{ 63 | Severity: hcl.DiagError, 64 | Summary: "Duplicate variable", 65 | Detail: "Stage with id " + stage.Id + " is defined more than once", 66 | }) 67 | } 68 | } 69 | } 70 | return diags 71 | } 72 | -------------------------------------------------------------------------------- /internal/ci/variable_hcl.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | func (v *Variable) Variables() []hcl.Traversal { 6 | var traversal []hcl.Traversal 7 | traversal = append(traversal, v.Value.Variables()...) 8 | return traversal 9 | 10 | } 11 | -------------------------------------------------------------------------------- /internal/ci/variable_parse.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/zclconf/go-cty/cty" 7 | "strings" 8 | ) 9 | 10 | func ParseVariableShell(raw string) (*Variable, hcl.Diagnostics) { 11 | var diags hcl.Diagnostics 12 | eq := strings.Index(raw, "=") 13 | if eq == -1 { 14 | diags = diags.Append(&hcl.Diagnostic{ 15 | Severity: hcl.DiagError, 16 | Summary: "Invalid -var option", 17 | Detail: fmt.Sprintf("The given -var option %q is not correctly specified. Must be a variable name and value separated by an equals sign, like -var=\"key=value\".", raw), 18 | }) 19 | return nil, diags 20 | } 21 | name := raw[:eq] 22 | rawVal := raw[eq+1:] 23 | if strings.HasSuffix(name, " ") { 24 | diags = diags.Append(&hcl.Diagnostic{ 25 | Severity: hcl.DiagError, 26 | Summary: "Invalid -var option", 27 | Detail: fmt.Sprintf("Variable name %q is invalid due to trailing space. Did you mean -var=\"%s=%s\"?", name, strings.TrimSuffix(name, " "), strings.TrimPrefix(rawVal, " ")), 28 | }) 29 | return nil, diags 30 | } 31 | return &Variable{ 32 | Id: name, 33 | Value: hcl.StaticExpr(cty.StringVal(rawVal), hcl.Range{}), 34 | }, diags 35 | 36 | } 37 | -------------------------------------------------------------------------------- /internal/ci/variable_retry.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | func (v *Variable) CanRetry() bool { 4 | return false 5 | } 6 | 7 | func (v *Variable) MinRetryBackoff() int { 8 | return 0 9 | } 10 | 11 | func (v *Variable) MaxRetryBackoff() int { 12 | return 0 13 | } 14 | 15 | func (v *Variable) RetryExponentialBackoff() bool { 16 | return false 17 | } 18 | 19 | func (v *Variable) MaxRetries() int { 20 | return 0 21 | } 22 | -------------------------------------------------------------------------------- /internal/ci/variable_schema.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | type Variable struct { 6 | Id string `hcl:"id,label" json:"id"` 7 | 8 | Desc string `hcl:"description,optional" json:"description"` 9 | DependsOn hcl.Expression `hcl:"depends_on,optional" json:"depends_on"` 10 | Value hcl.Expression `hcl:"value,optional" json:"value"` 11 | Default hcl.Expression `hcl:"default,optional" json:"default"` 12 | Ty hcl.Expression `hcl:"type,optional" json:"type"` 13 | } 14 | 15 | type Variables []*Variable 16 | -------------------------------------------------------------------------------- /internal/conductor/impl.go: -------------------------------------------------------------------------------- 1 | package conductor 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "github.com/sirupsen/logrus" 6 | "sync" 7 | ) 8 | 9 | type Conductor interface { 10 | Eval() Eval 11 | Logger() logrus.Ext1FieldLogger 12 | TempDir() string 13 | } 14 | 15 | type Eval interface { 16 | Context() *hcl.EvalContext 17 | Mutex() *sync.RWMutex 18 | } 19 | -------------------------------------------------------------------------------- /internal/core/togomak.go: -------------------------------------------------------------------------------- 1 | package core 2 | -------------------------------------------------------------------------------- /internal/dg/wrapper.go: -------------------------------------------------------------------------------- 1 | package dg 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "sync" 6 | ) 7 | 8 | type AbstractDiagnostics interface { 9 | Append(diag *hcl.Diagnostic) 10 | Extend(diags hcl.Diagnostics) 11 | HasErrors() bool 12 | Diagnostics() hcl.Diagnostics 13 | Error() string 14 | Errs() []error 15 | 16 | Safe() *SafeDiagnostics 17 | Unsafe() *Diagnostics 18 | } 19 | 20 | type SafeDiagnostics struct { 21 | diagsMu sync.Mutex 22 | diags hcl.Diagnostics 23 | 24 | expired bool 25 | } 26 | 27 | func (d *SafeDiagnostics) Append(diag *hcl.Diagnostic) { 28 | if d.expired { 29 | panic("Diagnostics expired") 30 | } 31 | d.diagsMu.Lock() 32 | defer d.diagsMu.Unlock() 33 | d.diags = d.diags.Append(diag) 34 | } 35 | 36 | func (d *SafeDiagnostics) Extend(diags hcl.Diagnostics) { 37 | if d.expired { 38 | panic("Diagnostics expired") 39 | } 40 | d.diagsMu.Lock() 41 | defer d.diagsMu.Unlock() 42 | d.diags = d.diags.Extend(diags) 43 | } 44 | 45 | func (d *SafeDiagnostics) HasErrors() bool { 46 | return d.diags.HasErrors() 47 | } 48 | 49 | func (d *SafeDiagnostics) Diagnostics() hcl.Diagnostics { 50 | return d.diags 51 | } 52 | 53 | func (d *SafeDiagnostics) Error() string { 54 | return d.diags.Error() 55 | } 56 | 57 | func (d *SafeDiagnostics) Errs() []error { 58 | return d.diags.Errs() 59 | } 60 | 61 | func (d *SafeDiagnostics) Unsafe() *Diagnostics { 62 | d.expired = true 63 | return &Diagnostics{ 64 | diags: d.diags, 65 | } 66 | } 67 | 68 | func (d *SafeDiagnostics) Safe() *SafeDiagnostics { 69 | return d 70 | } 71 | 72 | type Diagnostics struct { 73 | expired bool 74 | diags hcl.Diagnostics 75 | } 76 | 77 | func (d *Diagnostics) Append(diag *hcl.Diagnostic) { 78 | if d.expired { 79 | panic("Diagnostics expired") 80 | } 81 | d.diags = d.diags.Append(diag) 82 | } 83 | 84 | func (d *Diagnostics) Extend(diags hcl.Diagnostics) { 85 | if d.expired { 86 | panic("Diagnostics expired") 87 | } 88 | d.diags = d.diags.Extend(diags) 89 | } 90 | 91 | func (d *Diagnostics) HasErrors() bool { 92 | return d.diags.HasErrors() 93 | } 94 | 95 | func (d *Diagnostics) Diagnostics() hcl.Diagnostics { 96 | return d.diags 97 | } 98 | 99 | func (d *Diagnostics) Error() string { 100 | return d.diags.Error() 101 | } 102 | 103 | func (d *Diagnostics) Errs() []error { 104 | return d.diags.Errs() 105 | } 106 | 107 | func (d *Diagnostics) Safe() *SafeDiagnostics { 108 | d.expired = true 109 | return &SafeDiagnostics{ 110 | diags: d.diags, 111 | } 112 | } 113 | 114 | func (d *Diagnostics) Unsafe() *Diagnostics { 115 | return d 116 | } 117 | -------------------------------------------------------------------------------- /internal/dg/wrapper_test.go: -------------------------------------------------------------------------------- 1 | package dg 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2" 5 | "sync" 6 | "testing" 7 | ) 8 | 9 | func TestSafeDiagnostics_Append(t *testing.T) { 10 | // test for concurrent writes to the diagnostics 11 | 12 | d := &SafeDiagnostics{} 13 | var wg sync.WaitGroup 14 | for i := 0; i < 100; i++ { 15 | 16 | wg.Add(1) 17 | go func() { 18 | d.Append(&hcl.Diagnostic{ 19 | Severity: 0, 20 | Summary: "", 21 | Detail: "", 22 | Subject: nil, 23 | Context: nil, 24 | Expression: nil, 25 | EvalContext: nil, 26 | Extra: nil, 27 | }) 28 | wg.Done() 29 | }() 30 | } 31 | wg.Wait() 32 | if len(d.Diagnostics()) != 100 { 33 | t.Errorf("Diagnostics not written concurrently") 34 | } 35 | } 36 | 37 | func TestSafeDiagnostics_Extend(t *testing.T) { 38 | 39 | d := &SafeDiagnostics{} 40 | var wg sync.WaitGroup 41 | for i := 0; i < 100; i++ { 42 | 43 | wg.Add(1) 44 | go func() { 45 | d.Extend(hcl.Diagnostics{ 46 | { 47 | Severity: 0, 48 | Summary: "", 49 | Detail: "", 50 | Subject: nil, 51 | Context: nil, 52 | Expression: nil, 53 | EvalContext: nil, 54 | Extra: nil, 55 | }, 56 | }) 57 | wg.Done() 58 | }() 59 | } 60 | wg.Wait() 61 | if len(d.Diagnostics()) != 100 { 62 | t.Errorf("Diagnostics not written concurrently") 63 | } 64 | } 65 | 66 | func TestSafeDiagnostics_HasErrors(t *testing.T) { 67 | d := &SafeDiagnostics{} 68 | d.Extend(hcl.Diagnostics{ 69 | { 70 | Severity: hcl.DiagError, 71 | Summary: "", 72 | Detail: "", 73 | Subject: nil, 74 | Context: nil, 75 | Expression: nil, 76 | EvalContext: nil, 77 | Extra: nil, 78 | }, 79 | }) 80 | if !d.HasErrors() { 81 | t.Errorf("HasErrors not working") 82 | } 83 | } 84 | 85 | func TestSafeDiagnostics_Diagnostics(t *testing.T) { 86 | d := &SafeDiagnostics{} 87 | d.Extend(hcl.Diagnostics{ 88 | { 89 | Severity: 0, 90 | Summary: "", 91 | Detail: "", 92 | Subject: nil, 93 | Context: nil, 94 | Expression: nil, 95 | EvalContext: nil, 96 | Extra: nil, 97 | }, 98 | }) 99 | if len(d.Diagnostics()) != 1 { 100 | t.Errorf("Diagnostics not working") 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /internal/filter/filter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "fmt" 5 | "github.com/srevinsaju/togomak/v1/internal/blocks" 6 | "strings" 7 | ) 8 | 9 | type OperationTemp string 10 | 11 | const OperationTempRun OperationTemp = "" 12 | 13 | type Item struct { 14 | Id string 15 | Type string 16 | Operation OperationTemp 17 | } 18 | 19 | func (c Item) RunnableId() string { 20 | 21 | return fmt.Sprintf("%s.%s", c.Type, c.Id) 22 | } 23 | 24 | func (c Item) Identifier() string { 25 | return c.RunnableId() 26 | } 27 | 28 | type FilterList []Item 29 | 30 | func (c FilterList) Get(runnableId string) (FilterList, bool) { 31 | var stages []Item 32 | for _, stage := range c { 33 | if strings.HasPrefix(stage.Identifier(), runnableId) { 34 | stages = append(stages, stage) 35 | } 36 | } 37 | return stages, len(stages) > 0 38 | } 39 | 40 | func (c Item) Child() Item { 41 | return Item{ 42 | Id: c.Id[strings.IndexRune(c.Id, '.')+1:], 43 | Operation: c.Operation, 44 | Type: c.Type, 45 | } 46 | 47 | } 48 | 49 | func (c FilterList) Children(runnableId string) FilterList { 50 | var stages []Item 51 | for _, stage := range c { 52 | if strings.HasPrefix(stage.Identifier(), runnableId) && stage.Identifier() != runnableId { 53 | stages = append(stages, stage.Child()) 54 | } 55 | } 56 | return stages 57 | } 58 | 59 | func (c FilterList) AllOperations(operation OperationTemp) bool { 60 | for _, stage := range c { 61 | if stage.Operation != operation { 62 | return false 63 | } 64 | } 65 | return true 66 | } 67 | 68 | func (c FilterList) AnyOperations(operation OperationTemp) bool { 69 | for _, stage := range c { 70 | if stage.Operation == operation { 71 | return true 72 | } 73 | } 74 | return false 75 | } 76 | 77 | func (c FilterList) Marshall() []string { 78 | var stages []string 79 | for _, stage := range c { 80 | stages = append(stages, string(stage.Operation)+stage.Type+"."+stage.Id) 81 | } 82 | return stages 83 | } 84 | 85 | func (c FilterList) HasOperationType(operation OperationTemp) bool { 86 | for _, stage := range c { 87 | if stage.Operation == operation { 88 | return true 89 | } 90 | } 91 | return false 92 | } 93 | 94 | func NewFilterItem(arg string) Item { 95 | var operation OperationTemp 96 | 97 | ty := blocks.StageBlock 98 | 99 | if strings.HasPrefix(arg, string(OperationWhitelist)) { 100 | operation = OperationWhitelist 101 | } else if strings.HasPrefix(arg, string(OperationBlacklist)) { 102 | operation = OperationBlacklist 103 | } else if strings.HasPrefix(arg, string(OperationDaemonize)) { 104 | operation = OperationDaemonize 105 | } else if strings.HasPrefix(arg, string(OperationRunLifecycle)) { 106 | operation = OperationRunLifecycle 107 | ty = blocks.LifecycleBlock 108 | } else { 109 | operation = OperationTempRun 110 | } 111 | 112 | // TODO: improve this 113 | if strings.Contains("module.", arg) { 114 | ty = blocks.ModuleBlock 115 | } else if strings.Contains("stage.", arg) { 116 | ty = blocks.StageBlock 117 | } else if strings.Contains("lifecycle.", arg) { 118 | ty = blocks.LifecycleBlock 119 | } 120 | 121 | id := strings.TrimPrefix(arg, string(operation)) 122 | id = strings.TrimPrefix(id, ty+".") 123 | return Item{ 124 | Id: id, 125 | Type: ty, 126 | Operation: operation, 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /internal/filter/filter_model.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import "strings" 4 | 5 | type Operation int64 6 | 7 | const ( 8 | OperationInvalid Operation = -1 9 | 10 | OperationAdd Operation = iota 11 | OperationSubtract 12 | 13 | OperationRun 14 | ) 15 | 16 | func (op Operation) String() string { 17 | switch op { 18 | case OperationAdd: 19 | return "+" 20 | case OperationSubtract: 21 | return "^" 22 | case OperationRun: 23 | return "" 24 | } 25 | return "" 26 | } 27 | 28 | type Rule struct { 29 | // Type of block 30 | Type string 31 | 32 | // Operation is the operation that needs to be applied on the block 33 | Operation Operation 34 | 35 | // ID is the identifier of the block 36 | ID string 37 | } 38 | 39 | func Unmarshal(arg string) Rule { 40 | // Split the input string into parts using whitespace as the separator 41 | parts := strings.Fields(arg) 42 | 43 | // Initialize a Rule with default values 44 | rule := Rule{ 45 | Type: "", // Default Type 46 | Operation: OperationRun, // Default Operation 47 | ID: "", // Default ID 48 | } 49 | 50 | // If there are no parts, return the default rule 51 | if len(parts) == 0 { 52 | return rule 53 | } 54 | 55 | // Extract the Operation based on the first part of the input string 56 | switch parts[0] { 57 | case "+": 58 | rule.Operation = OperationAdd 59 | case "^": 60 | rule.Operation = OperationSubtract 61 | } 62 | 63 | // If there is only one part (the Operation), return the rule with the Operation set 64 | if len(parts) == 1 { 65 | return rule 66 | } 67 | 68 | // If there are two parts, set the Type to the second part 69 | rule.Type = parts[1] 70 | 71 | // If there is a third part, set the ID to it 72 | if len(parts) >= 3 { 73 | rule.ID = parts[2] 74 | } 75 | 76 | return rule 77 | } 78 | -------------------------------------------------------------------------------- /internal/filter/filter_unmarshal_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | -------------------------------------------------------------------------------- /internal/filter/operations.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | const ( 4 | OperationDaemonize OperationTemp = "&" 5 | OperationBlacklist OperationTemp = "^" 6 | OperationWhitelist OperationTemp = "+" 7 | 8 | OperationRunLifecycle OperationTemp = ":" 9 | ) 10 | -------------------------------------------------------------------------------- /internal/global/global.go: -------------------------------------------------------------------------------- 1 | package global 2 | -------------------------------------------------------------------------------- /internal/global/hcl.go: -------------------------------------------------------------------------------- 1 | package global 2 | -------------------------------------------------------------------------------- /internal/global/logging.go: -------------------------------------------------------------------------------- 1 | package global 2 | 3 | import "github.com/sirupsen/logrus" 4 | 5 | var logger = logrus.New() 6 | 7 | func SetLogger(l *logrus.Logger) { 8 | logger = l 9 | } 10 | 11 | func Logger() *logrus.Logger { 12 | return logger 13 | } 14 | -------------------------------------------------------------------------------- /internal/global/mutex.go: -------------------------------------------------------------------------------- 1 | package global 2 | 3 | import "sync" 4 | 5 | var ( 6 | DataBlockEvalContextMutex = sync.Mutex{} 7 | VariableBlockEvalContextMutex = sync.Mutex{} 8 | MacroBlockEvalContextMutex = sync.Mutex{} 9 | LocalBlockEvalContextMutex = sync.Mutex{} 10 | ) 11 | -------------------------------------------------------------------------------- /internal/log/logger.go: -------------------------------------------------------------------------------- 1 | package log 2 | -------------------------------------------------------------------------------- /internal/logging/google_cloud.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "context" 5 | "github.com/acarl005/stripansi" 6 | "github.com/sirupsen/logrus" 7 | "github.com/srevinsaju/togomak/v1/internal/meta" 8 | "github.com/srevinsaju/togomak/v1/internal/x" 9 | "google.golang.org/genproto/googleapis/api/monitoredres" 10 | "os" 11 | ) 12 | import "cloud.google.com/go/logging" 13 | 14 | var hostname string 15 | 16 | func googleCloudLoggingClient(project string) (*logging.Client, error) { 17 | loggerContext := context.Background() 18 | hostname = x.MustReturn(os.Hostname()).(string) 19 | 20 | // initialize the client 21 | client, err := logging.NewClient(loggerContext, project) 22 | return client, err 23 | } 24 | 25 | type GoogleCloudLoggerHook struct { 26 | client *logging.Client 27 | cfg Config 28 | project string 29 | } 30 | 31 | func NewGoogleCloudLoggerHook(cfg Config, project string) (*GoogleCloudLoggerHook, error) { 32 | client, err := googleCloudLoggingClient(project) 33 | return &GoogleCloudLoggerHook{cfg: cfg, client: client, project: project}, err 34 | } 35 | 36 | func (h *GoogleCloudLoggerHook) Fire(entry *logrus.Entry) error { 37 | // upload to google cloud logging 38 | // using google cloud API 39 | // https://cloud.google.com/logging/docs/reference/libraries#client-libraries-install-gow 40 | client := h.client 41 | logger := client.Logger(meta.AppName) 42 | severityLevel := logging.Default 43 | switch entry.Level { 44 | case logrus.DebugLevel: 45 | severityLevel = logging.Debug 46 | case logrus.InfoLevel: 47 | severityLevel = logging.Info 48 | case logrus.WarnLevel: 49 | severityLevel = logging.Warning 50 | case logrus.ErrorLevel: 51 | severityLevel = logging.Error 52 | case logrus.FatalLevel: 53 | severityLevel = logging.Critical 54 | case logrus.PanicLevel: 55 | severityLevel = logging.Alert 56 | } 57 | logger.Log(logging.Entry{ 58 | Payload: map[string]interface{}{ 59 | "message": stripansi.Strip(entry.Message), 60 | "labels": entry.Data, 61 | "app": meta.AppName, 62 | "version": meta.AppVersion, 63 | "host": hostname, 64 | }, 65 | Resource: &monitoredres.MonitoredResource{Type: "global"}, 66 | Trace: "togomak", 67 | Severity: severityLevel, 68 | Labels: map[string]string{ 69 | "app": meta.AppName, 70 | "version": meta.AppVersion, 71 | "instanceName": meta.AppName, 72 | "instanceId": h.cfg.CorrelationID, 73 | }, 74 | }) 75 | return nil 76 | } 77 | 78 | func (h *GoogleCloudLoggerHook) Levels() []logrus.Level { 79 | return []logrus.Level{logrus.InfoLevel, logrus.WarnLevel, logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel} 80 | } 81 | -------------------------------------------------------------------------------- /internal/logging/logging.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "errors" 5 | "github.com/rifflock/lfshook" 6 | "github.com/sirupsen/logrus" 7 | "github.com/urfave/cli/v2" 8 | "os" 9 | ) 10 | 11 | type Sink struct { 12 | Name string 13 | Level logrus.Level 14 | 15 | Options map[string]string 16 | } 17 | 18 | type Config struct { 19 | Verbosity int 20 | Child bool 21 | IsCI bool 22 | JSON bool 23 | CorrelationID string 24 | 25 | Sinks []Sink 26 | } 27 | 28 | func ParseSinksFromCLI(ctx *cli.Context) []Sink { 29 | var sinks []Sink 30 | file := ctx.Bool("logging.local.file") 31 | if file { 32 | sinks = append(sinks, Sink{ 33 | Name: "file", 34 | Level: logrus.DebugLevel, 35 | Options: map[string]string{ 36 | "path": ctx.String("logging.local.file.path"), 37 | }, 38 | }) 39 | } 40 | gcloud := ctx.Bool("logging.remote.google-cloud") 41 | if gcloud { 42 | sinks = append(sinks, Sink{ 43 | Name: "google-cloud", 44 | Level: logrus.DebugLevel, 45 | Options: map[string]string{ 46 | "project": ctx.String("logging.remote.google-cloud.project"), 47 | }, 48 | }) 49 | } 50 | return sinks 51 | } 52 | 53 | func New(cfg Config) (*logrus.Logger, error) { 54 | logger := logrus.New() 55 | logger.SetOutput(os.Stdout) 56 | logger.SetFormatter(&logrus.TextFormatter{ 57 | FullTimestamp: false, 58 | DisableTimestamp: cfg.Child, 59 | }) 60 | switch cfg.Verbosity { 61 | case -1: 62 | case 0: 63 | logger.SetLevel(logrus.InfoLevel) 64 | break 65 | case 1: 66 | logger.SetLevel(logrus.DebugLevel) 67 | break 68 | default: 69 | logger.SetLevel(logrus.TraceLevel) 70 | break 71 | } 72 | if cfg.IsCI { 73 | logger.SetFormatter(&logrus.TextFormatter{ 74 | DisableColors: false, 75 | EnvironmentOverrideColors: false, 76 | ForceColors: true, 77 | ForceQuote: false, 78 | }) 79 | } 80 | if cfg.Child { 81 | logger.SetFormatter(&logrus.TextFormatter{ 82 | DisableTimestamp: true, 83 | DisableColors: false, 84 | EnvironmentOverrideColors: false, 85 | ForceColors: true, 86 | ForceQuote: false, 87 | }) 88 | } 89 | if cfg.JSON { 90 | logger.SetFormatter(&logrus.JSONFormatter{}) 91 | } 92 | 93 | for _, sink := range cfg.Sinks { 94 | switch sink.Name { 95 | case "file": 96 | path, ok := sink.Options["path"] 97 | if !ok { 98 | path = "togomak.log" 99 | } 100 | hook := lfshook.NewHook(path, &logrus.JSONFormatter{}) 101 | logger.AddHook(hook) 102 | case "google-cloud": 103 | project, ok := sink.Options["project"] 104 | if !ok { 105 | return nil, errors.New("google-cloud sink requires project option") 106 | } 107 | hook, err := NewGoogleCloudLoggerHook(cfg, project) 108 | if err != nil { 109 | return nil, err 110 | } 111 | logger.AddHook(hook) 112 | default: 113 | return nil, errors.New("unknown sink: " + sink.Name) 114 | } 115 | } 116 | 117 | return logger, nil 118 | } 119 | -------------------------------------------------------------------------------- /internal/meta/app.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | var ( 4 | AppVersion = "v1.x" 5 | ) 6 | 7 | const ( 8 | AppName = "togomak" 9 | 10 | AppDescription = "A simple, declarative, and reproducible CI/CD pipeline generator powered by HCL" 11 | 12 | ConfigFileName = "togomak.hcl" 13 | BuildDirPrefix = ".togomak" 14 | 15 | EnvVarPrefix = "TOGOMAK__" 16 | 17 | OutputEnvFile = ".togomak.env" 18 | OutputEnvVar = "TOGOMAK_OUTPUTS" 19 | 20 | RootStage = "togomak.root" 21 | PreStage = "togomak.pre" 22 | PostStage = "togomak.post" 23 | ) 24 | -------------------------------------------------------------------------------- /internal/orchestra/context.go: -------------------------------------------------------------------------------- 1 | package orchestra 2 | -------------------------------------------------------------------------------- /internal/orchestra/format.go: -------------------------------------------------------------------------------- 1 | package orchestra 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/bmatcuk/doublestar" 7 | "github.com/hashicorp/hcl/v2/hclwrite" 8 | "github.com/srevinsaju/togomak/v1/internal/ci" 9 | "github.com/srevinsaju/togomak/v1/internal/parse" 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | func Format(cfg ci.ConductorConfig, check bool, recursive bool) error { 15 | conductor := ci.NewConductor(cfg) 16 | 17 | var toFormat []string 18 | 19 | if recursive { 20 | matches, err := doublestar.Glob("**/*.hcl") 21 | for _, path := range matches { 22 | conductor.Logger().Tracef("Found %s", path) 23 | data, err := os.ReadFile(path) 24 | if err != nil { 25 | return err 26 | } 27 | outSrc := hclwrite.Format(data) 28 | if !bytes.Equal(outSrc, data) { 29 | conductor.Logger().Tracef("%s needs formatting", path) 30 | toFormat = append(toFormat, path) 31 | } 32 | } 33 | if err != nil { 34 | conductor.Logger().Fatalf("Error while globbing for **/*.hcl: %s", err) 35 | } 36 | } else { 37 | fDir := parse.ConfigFileDir(conductor.Config.Paths) 38 | fNames, err := os.ReadDir(fDir) 39 | if err != nil { 40 | panic(err) 41 | } 42 | 43 | for _, f := range fNames { 44 | if f.IsDir() { 45 | continue 46 | } 47 | if filepath.Ext(f.Name()) != ".hcl" { 48 | continue 49 | } 50 | fn := filepath.Join(fDir, f.Name()) 51 | data, err := os.ReadFile(fn) 52 | if err != nil { 53 | return err 54 | } 55 | outSrc := hclwrite.Format(data) 56 | if !bytes.Equal(outSrc, data) { 57 | conductor.Logger().Tracef("%s needs formatting", fn) 58 | toFormat = append(toFormat, fn) 59 | } 60 | } 61 | } 62 | for _, fn := range toFormat { 63 | fmt.Println(fn) 64 | if !check { 65 | data, err := os.ReadFile(fn) 66 | if err != nil { 67 | panic(err) 68 | } 69 | outSrc := hclwrite.Format(data) 70 | err = os.WriteFile(fn, outSrc, 0644) 71 | if err != nil { 72 | panic(err) 73 | } 74 | } 75 | } 76 | if check && len(toFormat) > 0 { 77 | os.Exit(1) 78 | } 79 | return nil 80 | 81 | } 82 | -------------------------------------------------------------------------------- /internal/orchestra/imports.go: -------------------------------------------------------------------------------- 1 | package orchestra 2 | -------------------------------------------------------------------------------- /internal/orchestra/init.go: -------------------------------------------------------------------------------- 1 | package orchestra 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2/hclwrite" 5 | "github.com/srevinsaju/togomak/v1/internal/meta" 6 | "github.com/srevinsaju/togomak/v1/internal/ui" 7 | "github.com/srevinsaju/togomak/v1/internal/x" 8 | "github.com/zclconf/go-cty/cty" 9 | "os" 10 | "path/filepath" 11 | ) 12 | 13 | func InitPipeline(dir string) { 14 | f := hclwrite.NewEmptyFile() 15 | rootBody := f.Body() 16 | togomakBlock := rootBody.AppendNewBlock("togomak", nil) 17 | togomakBlock.Body().SetAttributeValue("version", cty.NumberIntVal(2)) 18 | 19 | // add the data block 20 | stageBlock := rootBody.AppendNewBlock("stage", []string{"example"}) 21 | stageBlock.Body().SetAttributeValue("name", cty.StringVal("example")) 22 | stageBlock.Body().SetAttributeValue("script", cty.StringVal("echo hello world")) 23 | 24 | path := filepath.Join(dir, meta.ConfigFileName) 25 | if x.FileExists(path) { 26 | allow := ui.PromptYesNo("A togomak pipeline already exists in this directory. Do you want to overwrite it?") 27 | if !allow { 28 | os.Exit(1) 29 | } 30 | x.Must(os.Remove(path)) 31 | } 32 | 33 | file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644) 34 | if err != nil { 35 | ui.Error("failed to create pipeline file") 36 | os.Exit(1) 37 | } 38 | defer file.Close() 39 | 40 | n, err := f.WriteTo(file) 41 | if err != nil { 42 | ui.Error("failed to write pipeline file") 43 | os.Exit(1) 44 | } 45 | ui.Success("successfully wrote %d bytes to %s", n, path) 46 | 47 | } 48 | -------------------------------------------------------------------------------- /internal/orchestra/list.go: -------------------------------------------------------------------------------- 1 | package orchestra 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/srevinsaju/togomak/v1/internal/ci" 7 | "github.com/srevinsaju/togomak/v1/internal/ui" 8 | "os" 9 | ) 10 | 11 | func List(cfg ci.ConductorConfig) error { 12 | 13 | conductor := ci.NewConductor(cfg) 14 | logger := conductor.Logger() 15 | 16 | dgwriter := hcl.NewDiagnosticTextWriter(os.Stdout, conductor.Parser.Files(), 0, true) 17 | pipe, hclDiags := ci.Read(conductor) 18 | if hclDiags.HasErrors() { 19 | logger.Fatal(dgwriter.WriteDiagnostics(hclDiags)) 20 | } 21 | 22 | pipe, d := pipe.ExpandImports(conductor, conductor.Config.Paths.Cwd) 23 | hclDiags = hclDiags.Extend(d) 24 | 25 | for _, stage := range pipe.Stages { 26 | fmt.Println(ui.Bold(stage.Id)) 27 | } 28 | return nil 29 | 30 | } 31 | -------------------------------------------------------------------------------- /internal/orchestra/logging.go: -------------------------------------------------------------------------------- 1 | package orchestra 2 | -------------------------------------------------------------------------------- /internal/orchestra/orchestra.go: -------------------------------------------------------------------------------- 1 | package orchestra 2 | 3 | import ( 4 | "context" 5 | "github.com/srevinsaju/togomak/v1/internal/blocks" 6 | "github.com/srevinsaju/togomak/v1/internal/ci" 7 | "strings" 8 | 9 | "github.com/zclconf/go-cty/cty" 10 | "os" 11 | ) 12 | 13 | func ExpandGlobalParams(conductor *ci.Conductor) { 14 | paramsGo := make(map[string]cty.Value) 15 | if conductor.Config.Behavior.Child.Enabled { 16 | m := make(map[string]string) 17 | for _, e := range os.Environ() { 18 | if i := strings.Index(e, "="); i >= 0 { 19 | if strings.HasPrefix(e[:i], ci.TogomakParamEnvVarPrefix) { 20 | m[e[:i]] = e[i+1:] 21 | } 22 | } 23 | } 24 | for k, v := range m { 25 | if ci.TogomakParamEnvVarRegex.MatchString(k) { 26 | paramsGo[ci.TogomakParamEnvVarRegex.FindStringSubmatch(k)[1]] = cty.StringVal(v) 27 | } 28 | } 29 | } 30 | conductor.Eval().Mutex().Lock() 31 | conductor.Eval().Context().Variables[blocks.ParamBlock] = cty.ObjectVal(paramsGo) 32 | conductor.Eval().Mutex().Unlock() 33 | } 34 | 35 | func Perform(conductor *ci.Conductor) int { 36 | ctx, cancel := context.WithCancel(conductor.Context()) 37 | defer cancel() 38 | conductor.Update(ci.ConductorWithContext(ctx)) 39 | 40 | logger := conductor.Logger().WithField("orchestra", "perform") 41 | logger.Debugf("starting watchdogs and signal handlers") 42 | ExpandGlobalParams(conductor) 43 | 44 | // parse the config file 45 | pipe, hclDiags := ci.Read(conductor) 46 | if hclDiags.HasErrors() { 47 | logger.Fatal(conductor.DiagWriter.WriteDiagnostics(hclDiags)) 48 | } 49 | 50 | h, d := pipe.Run(conductor) 51 | if d.HasErrors() { 52 | return h.Fatal() 53 | } 54 | return h.Ok() 55 | } 56 | -------------------------------------------------------------------------------- /internal/orchestra/utils.go: -------------------------------------------------------------------------------- 1 | package orchestra 2 | -------------------------------------------------------------------------------- /internal/parse/eval_context.go: -------------------------------------------------------------------------------- 1 | package parse 2 | -------------------------------------------------------------------------------- /internal/parse/parse.go: -------------------------------------------------------------------------------- 1 | package parse 2 | 3 | import ( 4 | "github.com/srevinsaju/togomak/v1/internal/meta" 5 | "github.com/srevinsaju/togomak/v1/internal/path" 6 | "path/filepath" 7 | ) 8 | 9 | // ConfigFilePath returns the path to the configuration file. If the path is not absolute, it is assumed to be 10 | // relative to the working directory 11 | // DEPRECATED: use configFileDir instead 12 | func ConfigFilePath(paths *path.Path) string { 13 | pipelineFilePath := paths.Pipeline 14 | if pipelineFilePath == "" { 15 | pipelineFilePath = meta.ConfigFileName 16 | } 17 | 18 | if filepath.IsAbs(pipelineFilePath) == false { 19 | pipelineFilePath = filepath.Join(paths.Owd, pipelineFilePath) 20 | } 21 | return pipelineFilePath 22 | } 23 | 24 | func ConfigFileDir(paths *path.Path) string { 25 | return filepath.Dir(ConfigFilePath(paths)) 26 | } 27 | -------------------------------------------------------------------------------- /internal/path/models.go: -------------------------------------------------------------------------------- 1 | package path 2 | 3 | type Path struct { 4 | // Pipeline is the path to the pipeline file 5 | Pipeline string 6 | 7 | // Owd is the original working directory 8 | Owd string 9 | 10 | // Cwd is the current working directory 11 | Cwd string 12 | 13 | // Module is the path to the module file 14 | Module string 15 | } 16 | -------------------------------------------------------------------------------- /internal/rules/model.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/hcl/v2" 6 | "github.com/srevinsaju/togomak/v1/internal/blocks" 7 | "regexp" 8 | "strings" 9 | ) 10 | 11 | type OperationType int 12 | 13 | const ( 14 | OperationTypeNone OperationType = iota 15 | OperationTypeAdd 16 | OperationTypeSub 17 | OperationTypeAnd 18 | ) 19 | 20 | var ( 21 | operationAddMatcher = regexp.MustCompile(`^\+([a-zA-Z0-9.\-:_]+)$`) 22 | operationSubMatcher = regexp.MustCompile(`^\^([a-zA-Z0-9.\-:_]+)$`) 23 | 24 | operationAndMatcher = regexp.MustCompile(`^([a-zA-Z0-9.\-:_]+)$`) 25 | ) 26 | 27 | var OperationTypes = []OperationType{ 28 | OperationTypeAdd, 29 | OperationTypeSub, 30 | OperationTypeAnd, 31 | } 32 | 33 | func (op OperationType) String() string { 34 | switch op { 35 | case OperationTypeAdd: 36 | return "+" 37 | case OperationTypeSub: 38 | return "^" 39 | case OperationTypeAnd: 40 | return "" 41 | } 42 | 43 | return "" 44 | } 45 | 46 | func (op OperationType) Matcher() *regexp.Regexp { 47 | switch op { 48 | case OperationTypeAdd: 49 | return operationAddMatcher 50 | case OperationTypeSub: 51 | return operationSubMatcher 52 | case OperationTypeAnd: 53 | return operationAndMatcher 54 | } 55 | panic(fmt.Sprintf("invalid operation type: %d", op)) 56 | } 57 | 58 | type Operation struct { 59 | op OperationType 60 | runnable string 61 | } 62 | 63 | type Operations []*Operation 64 | 65 | func NewOperation(op OperationType, runnable string) *Operation { 66 | return &Operation{ 67 | op: op, 68 | runnable: runnable, 69 | } 70 | } 71 | 72 | func (ops Operations) Children(runnableId string) Operations { 73 | childOps := make(Operations, 0) 74 | for _, op := range ops { 75 | genericParam := true 76 | for _, block := range []string{blocks.StageBlock, blocks.ModuleBlock, blocks.MacroBlock} { 77 | if strings.HasPrefix(op.runnable, fmt.Sprintf("%s.", block)) { 78 | genericParam = false 79 | } 80 | } 81 | if genericParam { 82 | childOps = append(childOps, op) 83 | } 84 | } 85 | return childOps 86 | } 87 | 88 | func (op *Operation) String() string { 89 | return fmt.Sprintf("%s%s", op.op.String(), op.runnable) 90 | } 91 | 92 | func (ops Operations) Marshall() []string { 93 | var s []string 94 | for _, op := range ops { 95 | s = append(s, op.String()) 96 | } 97 | return s 98 | } 99 | 100 | func (op *Operation) RunnableId() string { 101 | return op.runnable 102 | } 103 | 104 | func (op *Operation) Operation() OperationType { 105 | return op.op 106 | } 107 | 108 | func OperationUnmarshal(arg string) (*Operation, hcl.Diagnostics) { 109 | op := OperationTypeNone 110 | var diags hcl.Diagnostics 111 | 112 | for _, opType := range OperationTypes { 113 | matcher := opType.Matcher() 114 | if matcher.MatchString(arg) { 115 | op = opType 116 | break 117 | } 118 | } 119 | if op == OperationTypeNone { 120 | return nil, diags.Append(&hcl.Diagnostic{ 121 | Severity: hcl.DiagError, 122 | Summary: "invalid operation", 123 | Detail: fmt.Sprintf("invalid operation, no operation found in %s.", arg), 124 | }) 125 | } 126 | item := op.Matcher().FindStringSubmatch(arg) 127 | if item[1] == "" { 128 | return nil, diags.Append(&hcl.Diagnostic{ 129 | Severity: hcl.DiagError, 130 | Summary: "invalid operation", 131 | Detail: fmt.Sprintf("invalid operation, no stage, module or macro followed operation '%s', found in %s.", op.String(), arg), 132 | }) 133 | } 134 | return NewOperation(op, item[1]), nil 135 | } 136 | 137 | func Unmarshal(args []string) (ops Operations, diags hcl.Diagnostics) { 138 | // dslTokens := make([]string, 0) 139 | ops = make(Operations, len(args)) 140 | var d hcl.Diagnostics 141 | for i, arg := range args { 142 | ops[i], d = OperationUnmarshal(arg) 143 | diags = diags.Extend(d) 144 | } 145 | return ops, diags 146 | } 147 | -------------------------------------------------------------------------------- /internal/rules/operation_test.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestOperationUnmarshal(t *testing.T) { 9 | tests := []struct { 10 | input string 11 | expected Operation 12 | mustError bool 13 | }{ 14 | { 15 | input: "+foo", 16 | expected: Operation{ 17 | op: OperationTypeAdd, 18 | runnable: "foo", 19 | }, 20 | }, 21 | { 22 | input: "^foo", 23 | expected: Operation{ 24 | op: OperationTypeSub, 25 | runnable: "foo", 26 | }, 27 | }, 28 | { 29 | input: "foo", 30 | expected: Operation{ 31 | op: OperationTypeAnd, 32 | runnable: "foo", 33 | }, 34 | }, 35 | { 36 | input: "+", 37 | expected: Operation{}, 38 | mustError: true, 39 | }, 40 | { 41 | input: "^", 42 | expected: Operation{}, 43 | mustError: true, 44 | }, 45 | { 46 | input: "", 47 | expected: Operation{}, 48 | mustError: true, 49 | }, 50 | { 51 | input: "foo+", 52 | expected: Operation{}, 53 | mustError: true, 54 | }, 55 | { 56 | input: "foo^", 57 | expected: Operation{}, 58 | mustError: true, 59 | }, 60 | { 61 | input: "foo+bar", 62 | expected: Operation{}, 63 | mustError: true, 64 | }, 65 | { 66 | input: "foo^bar", 67 | expected: Operation{}, 68 | mustError: true, 69 | }, 70 | { 71 | input: "foo+bar^baz", 72 | expected: Operation{}, 73 | mustError: true, 74 | }, 75 | { 76 | input: "+stage.bar", 77 | expected: Operation{ 78 | op: OperationTypeAdd, 79 | runnable: "stage.bar", 80 | }, 81 | }, 82 | } 83 | 84 | for i, test := range tests { 85 | fmt.Println(i, test.input) 86 | op, d := OperationUnmarshal(test.input) 87 | if test.mustError && !d.HasErrors() { 88 | t.Errorf("expected error for '%s'", test.input) 89 | continue 90 | } 91 | if !test.mustError && d.HasErrors() { 92 | t.Errorf("unexpected error for '%s'", test.input) 93 | continue 94 | } 95 | if test.mustError && d.HasErrors() { 96 | continue 97 | } 98 | if op.op != test.expected.op { 99 | t.Errorf("expected op %s, got %s for '%s'", test.expected.op, op.op, test.input) 100 | continue 101 | } 102 | if op.runnable != test.expected.runnable { 103 | t.Errorf("expected runnable %s, got %s for '%s'", test.expected.runnable, op.runnable, test.input) 104 | continue 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /internal/runnable/options.go: -------------------------------------------------------------------------------- 1 | package runnable 2 | 3 | import ( 4 | "github.com/srevinsaju/togomak/v1/internal/behavior" 5 | "github.com/srevinsaju/togomak/v1/internal/path" 6 | "github.com/zclconf/go-cty/cty" 7 | ) 8 | 9 | type Config struct { 10 | Status *Status 11 | Parent *ParentConfig 12 | Hook bool 13 | 14 | Paths *path.Path 15 | 16 | Each map[string]cty.Value 17 | 18 | Behavior *behavior.Behavior 19 | } 20 | 21 | type ParentConfig struct { 22 | Id string 23 | Name string 24 | } 25 | 26 | type Option func(*Config) 27 | 28 | func WithStatus(status StatusType) Option { 29 | return func(c *Config) { 30 | c.Status = &Status{ 31 | Status: status, 32 | } 33 | } 34 | } 35 | 36 | func WithStatusOutput(artifact string) Option { 37 | return func(c *Config) { 38 | c.Status.Output = artifact 39 | } 40 | } 41 | 42 | func WithPaths(paths *path.Path) Option { 43 | return func(c *Config) { 44 | c.Paths = paths 45 | } 46 | } 47 | 48 | func WithParent(parent ParentConfig) Option { 49 | return func(c *Config) { 50 | c.Parent = &parent 51 | } 52 | } 53 | 54 | func WithEach(k cty.Value, v cty.Value) Option { 55 | return func(c *Config) { 56 | c.Each = map[string]cty.Value{ 57 | "key": k, 58 | "value": v, 59 | } 60 | } 61 | } 62 | 63 | func WithBehavior(behavior *behavior.Behavior) Option { 64 | return func(c *Config) { 65 | c.Behavior = behavior 66 | } 67 | } 68 | 69 | func NewDefaultConfig() *Config { 70 | return &Config{ 71 | Status: &Status{Status: StatusRunning}, 72 | Parent: nil, 73 | Hook: false, 74 | Paths: nil, 75 | Behavior: nil, 76 | } 77 | } 78 | 79 | func NewConfig(options ...Option) *Config { 80 | c := NewDefaultConfig() 81 | for _, option := range options { 82 | option(c) 83 | } 84 | return c 85 | } 86 | 87 | func WithHook() Option { 88 | return func(c *Config) { 89 | c.Hook = true 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /internal/runnable/status.go: -------------------------------------------------------------------------------- 1 | package runnable 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | type StatusType string 6 | 7 | const ( 8 | StatusSuccess StatusType = "success" 9 | StatusFailure StatusType = "failure" 10 | StatusTerminated StatusType = "terminated" 11 | StatusRunning StatusType = "running" 12 | StatusSkipped StatusType = "skipped" 13 | StatusUnknown StatusType = "unknown" 14 | ) 15 | 16 | func (s StatusType) String() string { 17 | return string(s) 18 | } 19 | 20 | type Status struct { 21 | // Diags is the diagnostics of the runnable 22 | Diags hcl.Diagnostics 23 | 24 | // Status is the status of the runnable 25 | Status StatusType 26 | 27 | Output string 28 | } 29 | -------------------------------------------------------------------------------- /internal/third-party/docker/docker.go: -------------------------------------------------------------------------------- 1 | package docker 2 | -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/README.md: -------------------------------------------------------------------------------- 1 | # terraform 2 | 3 | This directory is vendored from 4 | [terraform](https://github.com/hashicorp/terraform) v1.5.0. 5 | 6 | The following changes have been made: 7 | * Removed CIDR function 8 | * Replaced all `github.com/hashicorp/terraform` imports with `github.com/srevinsaju/togomak/v1/pkg/third-party/hashicorp/terraform/` 9 | 10 | This directory is licensed under the Mozilla Public License v2.0, as defined by Terraform. Respect it's [LICENSE](https://github.com/hashicorp/terraform/blob/main/LICENSE) file. 11 | -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/redact.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package funcs 5 | 6 | import ( 7 | "fmt" 8 | "github.com/srevinsaju/togomak/v1/internal/third-party/hashicorp/terraform/lang/marks" 9 | 10 | "github.com/zclconf/go-cty/cty" 11 | ) 12 | 13 | func redactIfSensitive(value interface{}, markses ...cty.ValueMarks) string { 14 | if marks.Has(cty.DynamicVal.WithMarks(markses...), marks.Sensitive) { 15 | return "(sensitive value)" 16 | } 17 | switch v := value.(type) { 18 | case string: 19 | return fmt.Sprintf("%q", v) 20 | default: 21 | return fmt.Sprintf("%v", v) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/redact_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package funcs 5 | 6 | import ( 7 | "github.com/srevinsaju/togomak/v1/internal/third-party/hashicorp/terraform/lang/marks" 8 | "testing" 9 | 10 | "github.com/zclconf/go-cty/cty" 11 | ) 12 | 13 | func TestRedactIfSensitive(t *testing.T) { 14 | testCases := map[string]struct { 15 | value interface{} 16 | marks []cty.ValueMarks 17 | want string 18 | }{ 19 | "sensitive string": { 20 | value: "foo", 21 | marks: []cty.ValueMarks{cty.NewValueMarks(marks.Sensitive)}, 22 | want: "(sensitive value)", 23 | }, 24 | "marked non-sensitive string": { 25 | value: "foo", 26 | marks: []cty.ValueMarks{cty.NewValueMarks("boop")}, 27 | want: `"foo"`, 28 | }, 29 | "sensitive string with other marks": { 30 | value: "foo", 31 | marks: []cty.ValueMarks{cty.NewValueMarks("boop"), cty.NewValueMarks(marks.Sensitive)}, 32 | want: "(sensitive value)", 33 | }, 34 | "sensitive number": { 35 | value: 12345, 36 | marks: []cty.ValueMarks{cty.NewValueMarks(marks.Sensitive)}, 37 | want: "(sensitive value)", 38 | }, 39 | "non-sensitive number": { 40 | value: 12345, 41 | marks: []cty.ValueMarks{}, 42 | want: "12345", 43 | }, 44 | } 45 | 46 | for name, tc := range testCases { 47 | t.Run(name, func(t *testing.T) { 48 | got := redactIfSensitive(tc.value, tc.marks...) 49 | if got != tc.want { 50 | t.Errorf("wrong result, got %v, want %v", got, tc.want) 51 | } 52 | }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/refinements.go: -------------------------------------------------------------------------------- 1 | package funcs 2 | 3 | import ( 4 | "github.com/zclconf/go-cty/cty" 5 | ) 6 | 7 | func refineNotNull(b *cty.RefinementBuilder) *cty.RefinementBuilder { 8 | return b.NotNull() 9 | } 10 | -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/sensitive.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package funcs 5 | 6 | import ( 7 | "github.com/srevinsaju/togomak/v1/internal/third-party/hashicorp/terraform/lang/marks" 8 | "github.com/zclconf/go-cty/cty" 9 | "github.com/zclconf/go-cty/cty/function" 10 | ) 11 | 12 | // SensitiveFunc returns a value identical to its argument except that 13 | // Terraform will consider it to be sensitive. 14 | var SensitiveFunc = function.New(&function.Spec{ 15 | Params: []function.Parameter{ 16 | { 17 | Name: "value", 18 | Type: cty.DynamicPseudoType, 19 | AllowUnknown: true, 20 | AllowNull: true, 21 | AllowMarked: true, 22 | AllowDynamicType: true, 23 | }, 24 | }, 25 | Type: func(args []cty.Value) (cty.Type, error) { 26 | // This function only affects the value's marks, so the result 27 | // type is always the same as the argument type. 28 | return args[0].Type(), nil 29 | }, 30 | Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { 31 | val, _ := args[0].Unmark() 32 | return val.Mark(marks.Sensitive), nil 33 | }, 34 | }) 35 | 36 | // NonsensitiveFunc takes a sensitive value and returns the same value without 37 | // the sensitive marking, effectively exposing the value. 38 | var NonsensitiveFunc = function.New(&function.Spec{ 39 | Params: []function.Parameter{ 40 | { 41 | Name: "value", 42 | Type: cty.DynamicPseudoType, 43 | AllowUnknown: true, 44 | AllowNull: true, 45 | AllowMarked: true, 46 | AllowDynamicType: true, 47 | }, 48 | }, 49 | Type: func(args []cty.Value) (cty.Type, error) { 50 | // This function only affects the value's marks, so the result 51 | // type is always the same as the argument type. 52 | return args[0].Type(), nil 53 | }, 54 | Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { 55 | if args[0].IsKnown() && !args[0].HasMark(marks.Sensitive) { 56 | return cty.DynamicVal, function.NewArgErrorf(0, "the given value is not sensitive, so this call is redundant") 57 | } 58 | v, m := args[0].Unmark() 59 | delete(m, marks.Sensitive) // remove the sensitive marking 60 | return v.WithMarks(m), nil 61 | }, 62 | }) 63 | 64 | func Sensitive(v cty.Value) (cty.Value, error) { 65 | return SensitiveFunc.Call([]cty.Value{v}) 66 | } 67 | 68 | func Nonsensitive(v cty.Value) (cty.Value, error) { 69 | return NonsensitiveFunc.Call([]cty.Value{v}) 70 | } 71 | -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/testdata/bare.tmpl: -------------------------------------------------------------------------------- 1 | ${val} -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/testdata/func.tmpl: -------------------------------------------------------------------------------- 1 | The items are ${join(", ", list)} -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/testdata/hello.tmpl: -------------------------------------------------------------------------------- 1 | Hello, ${name}! -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/testdata/hello.txt: -------------------------------------------------------------------------------- 1 | Hello World -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/testdata/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srevinsaju/togomak/54334bd492d502b139492f63282323d6b9ee8c13/internal/third-party/hashicorp/terraform/lang/funcs/testdata/icon.png -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/testdata/list.tmpl: -------------------------------------------------------------------------------- 1 | %{ for x in list ~} 2 | - ${x} 3 | %{ endfor ~} 4 | -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/testdata/recursive.tmpl: -------------------------------------------------------------------------------- 1 | ${templatefile("recursive.tmpl", {})} -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/funcs/testdata/unreadable/foobar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srevinsaju/togomak/54334bd492d502b139492f63282323d6b9ee8c13/internal/third-party/hashicorp/terraform/lang/funcs/testdata/unreadable/foobar -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/marks/marks.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package marks 5 | 6 | import ( 7 | "github.com/zclconf/go-cty/cty" 8 | ) 9 | 10 | // valueMarks allow creating strictly typed values for use as cty.Value marks. 11 | // Each distinct mark value must be a constant in this package whose value 12 | // is a valueMark whose underlying string matches the name of the variable. 13 | type valueMark string 14 | 15 | func (m valueMark) GoString() string { 16 | return "marks." + string(m) 17 | } 18 | 19 | // Has returns true if and only if the cty.Value has the given mark. 20 | func Has(val cty.Value, mark valueMark) bool { 21 | return val.HasMark(mark) 22 | } 23 | 24 | // Contains returns true if the cty.Value or any any value within it contains 25 | // the given mark. 26 | func Contains(val cty.Value, mark valueMark) bool { 27 | ret := false 28 | cty.Walk(val, func(_ cty.Path, v cty.Value) (bool, error) { 29 | if v.HasMark(mark) { 30 | ret = true 31 | return false, nil 32 | } 33 | return true, nil 34 | }) 35 | return ret 36 | } 37 | 38 | // Sensitive indicates that this value is marked as sensitive in the context of 39 | // Terraform. 40 | const Sensitive = valueMark("Sensitive") 41 | 42 | // TypeType is used to indicate that the value contains a representation of 43 | // another value's type. This is part of the implementation of the console-only 44 | // `type` function. 45 | const TypeType = valueMark("TypeType") 46 | -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/types/type_type.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package types 5 | 6 | import ( 7 | "reflect" 8 | 9 | "github.com/zclconf/go-cty/cty" 10 | ) 11 | 12 | // TypeType is a capsule type used to represent a cty.Type as a cty.Value. This 13 | // is used by the `type()` console function to smuggle cty.Type values to the 14 | // REPL session, where it can be displayed to the user directly. 15 | var TypeType = cty.Capsule("type", reflect.TypeOf(cty.Type{})) 16 | -------------------------------------------------------------------------------- /internal/third-party/hashicorp/terraform/lang/types/types.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | // Package types contains non-standard cty types used only within Terraform. 5 | package types 6 | -------------------------------------------------------------------------------- /internal/ui/getter.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import ( 4 | "fmt" 5 | "github.com/sirupsen/logrus" 6 | "github.com/srevinsaju/togomak/v1/internal/x" 7 | "io" 8 | ) 9 | 10 | // GetterProgressBar is a progress bar implementation for terraform downloads 11 | type GetterProgressBar struct { 12 | Logger *logrus.Entry 13 | src string 14 | pb *ProgressWriter 15 | } 16 | 17 | func NewGetterProgressBar(logger *logrus.Entry, src string) *GetterProgressBar { 18 | return &GetterProgressBar{ 19 | Logger: logger, 20 | src: src, 21 | } 22 | } 23 | 24 | // Init initializes the progress bar 25 | func (e *GetterProgressBar) Init() *GetterProgressBar { 26 | e.pb = NewProgressWriter(e.Logger, fmt.Sprintf("downloading %s", e.src)) 27 | return e 28 | } 29 | 30 | // TrackProgress tracks the progress of the download using the default ui.ProgressWriter implementation 31 | func (e *GetterProgressBar) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) (body io.ReadCloser) { 32 | for { 33 | _, err := io.CopyN(e.pb, stream, 1) 34 | if err != nil { 35 | x.Must(e.pb.Close()) 36 | return stream 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /internal/ui/passive.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import "github.com/sirupsen/logrus" 4 | 5 | type PassiveProgressBar struct { 6 | Logger *logrus.Entry 7 | Message string 8 | 9 | pb *ProgressWriter 10 | completer chan bool 11 | } 12 | 13 | func NewPassiveProgressBar(logger *logrus.Entry, message string) *PassiveProgressBar { 14 | return &PassiveProgressBar{ 15 | Logger: logger, 16 | Message: message, 17 | } 18 | } 19 | 20 | func (p *PassiveProgressBar) Init() { 21 | p.completer = make(chan bool) 22 | go p.run() 23 | } 24 | 25 | func (p *PassiveProgressBar) run() { 26 | p.pb = NewProgressWriter(p.Logger, p.Message) 27 | for { 28 | select { 29 | case <-p.completer: 30 | p.pb.Close() 31 | return 32 | default: 33 | p.pb.printProgress() 34 | } 35 | } 36 | } 37 | 38 | func (p *PassiveProgressBar) Done() { 39 | p.completer <- true 40 | } 41 | -------------------------------------------------------------------------------- /internal/ui/progress.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import ( 4 | "fmt" 5 | "github.com/bcicen/jstream" 6 | "github.com/sirupsen/logrus" 7 | "io" 8 | "time" 9 | ) 10 | 11 | type DockerProgressWriter struct { 12 | reader io.Reader 13 | bytesRead int64 14 | verb string 15 | created time.Time 16 | lastWritten time.Time 17 | status string 18 | lastStatus string 19 | writer io.Writer 20 | } 21 | 22 | type ProgressWriter struct { 23 | bytesRead int64 24 | verb string 25 | created time.Time 26 | lastWritten time.Time 27 | logger *logrus.Entry 28 | } 29 | 30 | func NewDockerProgressWriter(reader io.Reader, writer io.Writer, verb string) *DockerProgressWriter { 31 | return &DockerProgressWriter{ 32 | verb: verb, 33 | reader: reader, 34 | created: time.Now(), 35 | lastWritten: time.Now().Add(-time.Second * 5), 36 | writer: writer, 37 | status: "", 38 | lastStatus: "", 39 | } 40 | } 41 | 42 | func NewProgressWriter(logger *logrus.Entry, verb string) *ProgressWriter { 43 | return &ProgressWriter{ 44 | bytesRead: 0, 45 | verb: verb, 46 | created: time.Now(), 47 | lastWritten: time.Now().Add(-time.Second * 5), 48 | logger: logger, 49 | } 50 | } 51 | 52 | type Message struct { 53 | Status string `json:"status,omitempty"` 54 | } 55 | 56 | func (pr *DockerProgressWriter) Write(p []byte) (int, error) { 57 | d := jstream.NewDecoder(pr.reader, 0) 58 | 59 | for mv := range d.Stream() { 60 | m, ok := mv.Value.(map[string]interface{}) 61 | 62 | if ok && m["status"] != nil { 63 | pr.status = m["status"].(string) 64 | pr.printProgress() 65 | } 66 | } 67 | return d.Pos(), nil 68 | } 69 | 70 | func (pr *ProgressWriter) Write(p []byte) (int, error) { 71 | err := pr.printProgress() 72 | pr.bytesRead += int64(len(p)) 73 | return len(p), err 74 | } 75 | 76 | func (pr *DockerProgressWriter) printProgress() error { 77 | elapsed := time.Since(pr.lastWritten) 78 | var err error 79 | if elapsed.Seconds() >= 5 || pr.status != pr.lastStatus { 80 | pr.lastWritten = time.Now() 81 | pr.lastStatus = pr.status 82 | _, err = fmt.Fprintf(pr.writer, "Still %s... (%s) %s\n", pr.verb, pr.status, Bold(fmt.Sprintf("[%s elapsed]", time.Since(pr.created).Round(time.Second).String()))) 83 | } 84 | return err 85 | } 86 | 87 | func (pr *ProgressWriter) printProgress() error { 88 | elapsed := time.Since(pr.lastWritten) 89 | var err error 90 | if elapsed.Seconds() >= 5 { 91 | pr.lastWritten = time.Now() 92 | pr.logger.Infof("Still %s... %s\n", pr.verb, Bold(fmt.Sprintf("[%s elapsed]", time.Since(pr.created).Round(time.Second).String()))) 93 | } 94 | return err 95 | } 96 | 97 | func (pr *DockerProgressWriter) Close() error { 98 | _, err := fmt.Fprintf(pr.writer, "Completed %s in %s\n", pr.verb, Bold(fmt.Sprintf("[%s elapsed]", time.Since(pr.created).Round(time.Second).String()))) 99 | return err 100 | } 101 | 102 | func (pr *ProgressWriter) Close() error { 103 | pr.logger.Infof("Completed %s in %s\n", pr.verb, Bold(fmt.Sprintf("[%s elapsed]", time.Since(pr.created).Round(time.Second).String()))) 104 | return nil 105 | } 106 | -------------------------------------------------------------------------------- /internal/ui/prompt.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import "github.com/AlecAivazis/survey/v2" 4 | 5 | func PromptYesNo(text string) bool { 6 | // show a prompt to the user using survey 7 | // return true if the user says yes 8 | 9 | var result bool 10 | err := survey.AskOne(&survey.Confirm{ 11 | Message: text, 12 | }, &result) 13 | if err != nil { 14 | panic(err) 15 | } 16 | return result 17 | } 18 | -------------------------------------------------------------------------------- /internal/x/blocks.go: -------------------------------------------------------------------------------- 1 | package x 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func RenderBlock(block ...string) string { 8 | return strings.Join(block, ".") 9 | } 10 | -------------------------------------------------------------------------------- /internal/x/error.go: -------------------------------------------------------------------------------- 1 | package x 2 | 3 | func Must(err error) { 4 | if err != nil { 5 | panic(err) 6 | } 7 | } 8 | 9 | func MustReturn(v any, err error) any { 10 | if err != nil { 11 | panic(err) 12 | } 13 | return v 14 | } 15 | -------------------------------------------------------------------------------- /internal/x/error_test.go: -------------------------------------------------------------------------------- 1 | package x 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestMust(t *testing.T) { 9 | defer func() { 10 | if r := recover(); r == nil { 11 | t.Errorf("The code did not panic") 12 | } 13 | }() 14 | 15 | Must(nil) 16 | Must(fmt.Errorf("test")) 17 | } 18 | -------------------------------------------------------------------------------- /internal/x/file.go: -------------------------------------------------------------------------------- 1 | package x 2 | 3 | import "os" 4 | 5 | func FileExists(path string) bool { 6 | f, err := os.Stat(path) 7 | if os.IsNotExist(err) { 8 | return false 9 | } else { 10 | return !f.IsDir() 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | coverage_* 2 | coverage.out 3 | togomak_coverage 4 | .togomak 5 | -------------------------------------------------------------------------------- /tests/prompt_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "github.com/AlecAivazis/survey/v2/terminal" 6 | expect "github.com/Netflix/go-expect" 7 | pseudotty "github.com/creack/pty" 8 | "github.com/hinshun/vt10x" 9 | "log" 10 | "os" 11 | "os/exec" 12 | "testing" 13 | ) 14 | 15 | type expectConsole interface { 16 | ExpectString(string) 17 | ExpectEOF() 18 | SendLine(string) 19 | Send(string) 20 | Console() *expect.Console 21 | } 22 | 23 | type consoleWithErrorHandling struct { 24 | console *expect.Console 25 | t *testing.T 26 | } 27 | 28 | func (c *consoleWithErrorHandling) ExpectString(s string) { 29 | if _, err := c.console.ExpectString(s); err != nil { 30 | c.t.Helper() 31 | c.t.Fatalf("ExpectString(%q) = %v", s, err) 32 | } 33 | } 34 | 35 | func (c *consoleWithErrorHandling) SendLine(s string) { 36 | if _, err := c.console.SendLine(s); err != nil { 37 | c.t.Helper() 38 | c.t.Fatalf("SendLine(%q) = %v", s, err) 39 | } 40 | } 41 | 42 | func (c *consoleWithErrorHandling) Send(s string) { 43 | if _, err := c.console.Send(s); err != nil { 44 | c.t.Helper() 45 | c.t.Fatalf("Send(%q) = %v", s, err) 46 | } 47 | } 48 | 49 | func (c *consoleWithErrorHandling) ExpectEOF() { 50 | if _, err := c.console.ExpectEOF(); err != nil { 51 | c.t.Helper() 52 | c.t.Fatalf("ExpectEOF() = %v", err) 53 | } 54 | } 55 | 56 | func (c *consoleWithErrorHandling) Console() *expect.Console { 57 | return c.console 58 | } 59 | 60 | func RunTest(t *testing.T, procedure func(expectConsole), test func(terminal.Stdio) error) { 61 | t.Helper() 62 | t.Parallel() 63 | 64 | pty, tty, err := pseudotty.Open() 65 | if err != nil { 66 | t.Fatalf("failed to open pseudotty: %v", err) 67 | } 68 | 69 | term := vt10x.New(vt10x.WithWriter(tty)) 70 | c, err := expect.NewConsole(expect.WithStdin(pty), expect.WithStdout(term), expect.WithCloser(pty, tty)) 71 | if err != nil { 72 | t.Fatalf("failed to create console: %v", err) 73 | } 74 | defer c.Close() 75 | 76 | donec := make(chan struct{}) 77 | go func() { 78 | defer close(donec) 79 | procedure(&consoleWithErrorHandling{console: c, t: t}) 80 | }() 81 | 82 | stdio := terminal.Stdio{In: c.Tty(), Out: c.Tty(), Err: c.Tty()} 83 | if err := test(stdio); err != nil { 84 | t.Error(err) 85 | } 86 | 87 | if err := c.Tty().Close(); err != nil { 88 | t.Errorf("error closing Tty: %v", err) 89 | } 90 | <-donec 91 | } 92 | 93 | func TestPrompt(t *testing.T) { 94 | c, err := expect.NewConsole(expect.WithStdout(os.Stdout)) 95 | if err != nil { 96 | log.Fatal(err) 97 | } 98 | defer c.Close() 99 | 100 | cmd := exec.Command("./togomak_coverage", "-C", "../examples/prompt", "--ci=false") 101 | 102 | RunTest(t, func(c expectConsole) { 103 | c.SendLine("Shinji Ikari\n\n") 104 | fmt.Println(c.Console().ExpectEOF()) 105 | }, func(stdio terminal.Stdio) error { 106 | cmd.Stdin = stdio.In 107 | cmd.Stdout = stdio.Out 108 | cmd.Stderr = stdio.Out 109 | cmd.Env = append(os.Environ(), "QUIT_IF_NOT_SHINJI=true", fmt.Sprintf("GOCOVERDIR=%s", os.Getenv("PROMPT_GOCOVERDIR"))) 110 | return cmd.Start() 111 | 112 | }) 113 | 114 | err = cmd.Wait() 115 | if err != nil { 116 | d, err := os.ReadFile("/tmp/quit") 117 | if err != nil { 118 | panic(err) 119 | } 120 | fmt.Println(string(d)) 121 | t.Error(err) 122 | } 123 | fmt.Println(cmd.ProcessState.ExitCode()) 124 | 125 | } 126 | 127 | func TestInterrupt(t *testing.T) { 128 | 129 | } 130 | -------------------------------------------------------------------------------- /tests/tests/failing/cant-run-fail/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | stage "example" { 5 | if = this.what 6 | name = "example" 7 | script = "echo hello world" 8 | } 9 | -------------------------------------------------------------------------------- /tests/tests/failing/dependency-cycles/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "example_a" { 6 | depends_on = [stage.example_b] 7 | script = "echo hello world" 8 | } 9 | 10 | stage "example_b" { 11 | depends_on = [stage.example_a] 12 | script = "echo hello again" 13 | } 14 | -------------------------------------------------------------------------------- /tests/tests/failing/dependency-fail/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "test" { 6 | script = "exit 1" 7 | } 8 | 9 | stage "example" { 10 | depends_on = [stage.test] 11 | script = "exit 0" 12 | } 13 | -------------------------------------------------------------------------------- /tests/tests/failing/env-data-key-missing/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | data "env" "hello" { 6 | } 7 | 8 | stage "example" { 9 | name = "example" 10 | script = "echo hello world ${data.env.hello.value}" 11 | } 12 | -------------------------------------------------------------------------------- /tests/tests/failing/failing-hooks/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | 6 | stage "example" { 7 | script = "echo hello world && exit 1" 8 | 9 | pre_hook { 10 | stage "echo" { 11 | script = "echo before the script for stage ${this.id} runs" 12 | } 13 | } 14 | 15 | post_hook { 16 | stage "echo" { 17 | script = "echo the script for ${this.id} done with status ${this.status}" 18 | } 19 | } 20 | } 21 | 22 | stage "example_2" { 23 | script = "echo bye_world && exit 1" 24 | 25 | pre_hook { 26 | stage "echo" { 27 | script = "echo before the script for stage ${this.id} runs" 28 | } 29 | } 30 | 31 | post_hook { 32 | stage "echo" { 33 | script = "echo the script for ${this.id} done with status ${this.status}" 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tests/tests/failing/invalid-block-id/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | data "env" "hello_world" { 6 | key = "HOME" 7 | default = "@" 8 | } 9 | 10 | stage "example" { 11 | script = "echo ${data.env.hello_world.value}" 12 | } 13 | 14 | stage "example_2" { 15 | script = "echo ${data.env.hello_hello_world.value}" 16 | } 17 | 18 | -------------------------------------------------------------------------------- /tests/tests/failing/invalid-env-output/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "agent" { 6 | script = <<-EOT 7 | set -u 8 | echo "Ryoji Kaji" >> $TOGOMAK_OUTPUTS 9 | EOT 10 | } 11 | 12 | stage "seele" { 13 | depends_on = [stage.agent] 14 | name = "seele" 15 | script = "echo The agent from Seele reporting! ${output.AGENT}" 16 | } 17 | -------------------------------------------------------------------------------- /tests/tests/failing/invalid-path-import/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | stage "example" { 5 | name = "example" 6 | script = "echo hello world" 7 | } 8 | import { 9 | source = "git::ssh://git@github.com:codespaces/thisrepo.will-never-exist" 10 | } 11 | -------------------------------------------------------------------------------- /tests/tests/failing/invalid-stage-macro-invalid-file/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | macro "pen_pen" { 5 | source = "pen_pen.hcl" 6 | } 7 | stage "example" { 8 | use { 9 | macro = macro.pen_pen 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/tests/failing/invalid-stage-multiple-macros/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | macro "rei" { 5 | stage "this" { 6 | script = "hi, im rei" 7 | } 8 | } 9 | 10 | macro "gendo" { 11 | stage "this" { 12 | script = "hi, im gendo" 13 | } 14 | } 15 | 16 | stage "example" { 17 | use { 18 | macro = [macro.rei, macro.gendo] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tests/tests/failing/nested-failing-tests/nested.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | stage "test" { 6 | script = "echo hello world" 7 | } 8 | 9 | stage "failing" { 10 | depends_on = [ 11 | stage.test 12 | ] 13 | script = "echo failed && exit 1" 14 | } -------------------------------------------------------------------------------- /tests/tests/failing/nested-failing-tests/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | macro "nested" { 6 | files = { 7 | "togomak.hcl" : file("./nested.hcl") 8 | } 9 | } 10 | 11 | stage "example" { 12 | use { 13 | macro = macro.nested 14 | } 15 | } 16 | 17 | stage "another" { 18 | depends_on = [stage.example] 19 | script = "echo done" 20 | } -------------------------------------------------------------------------------- /tests/tests/failing/no-script-no-args/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | stage "example" { 5 | } 6 | -------------------------------------------------------------------------------- /tests/tests/failing/retry/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | 6 | stage "example" { 7 | retry { 8 | enabled = true 9 | exponential_backoff = true 10 | attempts = 3 11 | min_backoff = 1 12 | max_backoff = 3 13 | } 14 | script = "echo hello world && exit 1" 15 | } 16 | -------------------------------------------------------------------------------- /tests/tests/failing/self-referenced-dependencies/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | stage "example" { 5 | depends_on = [stage.example] 6 | name = "example" 7 | script = "echo hello world" 8 | } 9 | -------------------------------------------------------------------------------- /tests/togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | 6 | stage "build" { 7 | name = "build" 8 | dir = ".." 9 | script = "go build -cover -o tests/togomak_coverage ./cmd/togomak" 10 | } 11 | 12 | locals { 13 | coverage_data_dir = "${cwd}/coverage_data_files" 14 | coverage_merge_dir = "${cwd}/coverage_merge_dir" 15 | coverage_data_interactive_dir = "${cwd}/coverage_data_interactive_dir" 16 | } 17 | 18 | stage "coverage_prepare" { 19 | script = <<-EOT 20 | set -e 21 | rm -rf ${local.coverage_data_dir} && mkdir ${local.coverage_data_dir} 22 | rm -rf ${local.coverage_data_interactive_dir} && mkdir ${local.coverage_data_interactive_dir} 23 | rm -rf ${local.coverage_merge_dir} && mkdir ${local.coverage_merge_dir} 24 | EOT 25 | } 26 | 27 | stage "tests" { 28 | pre_hook { 29 | stage { 30 | script = "echo ${ansi.fg.green}${each.key}${ansi.reset}: full" 31 | } 32 | } 33 | 34 | depends_on = [stage.build, stage.coverage_prepare] 35 | for_each = fileset(cwd, "../examples/*/togomak.hcl") 36 | args = [ 37 | "./togomak_coverage", 38 | "-C", dirname(each.key), 39 | "--ci", "-v", "-v", "-v", 40 | ] 41 | 42 | env { 43 | name = "GOCOVERDIR" 44 | value = local.coverage_data_dir 45 | } 46 | env { 47 | name = "TOGOMAK_VAR_name" 48 | value = "bot" 49 | } 50 | } 51 | 52 | stage "module_phase_test" { 53 | pre_hook { 54 | stage { 55 | script = "echo ${ansi.fg.green}${each.key}${ansi.reset}: full" 56 | } 57 | } 58 | 59 | for_each = toset([ 60 | "../examples/module-phases-inheritance/togomak.hcl", 61 | ]) 62 | 63 | depends_on = [stage.build, stage.coverage_prepare] 64 | args = [ 65 | "./togomak_coverage", 66 | "-C", dirname(each.key), 67 | "--ci", "-v", "-v", "-v", 68 | "add" 69 | ] 70 | 71 | env { 72 | name = "GOCOVERDIR" 73 | value = local.coverage_data_dir 74 | } 75 | env { 76 | name = "TOGOMAK_VAR_name" 77 | value = "bot" 78 | } 79 | 80 | } 81 | 82 | 83 | stage "tests_dry_run" { 84 | pre_hook { 85 | stage { 86 | script = "echo ${ansi.fg.green}${each.key}${ansi.reset}: dry" 87 | } 88 | } 89 | 90 | depends_on = [stage.build, stage.coverage_prepare] 91 | for_each = fileset(cwd, "../examples/*/togomak.hcl") 92 | args = [ 93 | "./togomak_coverage", 94 | "-C", dirname(each.key), 95 | "--ci", "-v", "-v", "-v", "-n", 96 | ] 97 | 98 | env { 99 | name = "GOCOVERDIR" 100 | value = local.coverage_data_dir 101 | } 102 | 103 | env { 104 | name = "TOGOMAK_VAR_name" 105 | value = "bot" 106 | } 107 | } 108 | 109 | stage "fmt" { 110 | depends_on = [stage.build, stage.coverage_prepare] 111 | script = "./togomak_coverage fmt --check --recursive" 112 | } 113 | 114 | stage "cache" { 115 | depends_on = [stage.fmt, stage.tests, stage.tests_dry_run, stage.module_phase_test] 116 | script = "./togomak_coverage cache clean --recursive" 117 | } 118 | 119 | 120 | stage "failing" { 121 | depends_on = [stage.cache] 122 | for_each = fileset(cwd, "tests/failing/*/togomak.hcl") 123 | script = <<-EOT 124 | set +e 125 | ./togomak_coverage -C "${dirname(each.key)}" --ci -v -v -v 126 | result=$? 127 | if [ $result -eq 0 ]; then 128 | set -e 129 | echo "$i completed successfully when it was supposed to fail" 130 | exit 1 131 | fi 132 | EOT 133 | env { 134 | name = "GOCOVERDIR" 135 | value = local.coverage_data_dir 136 | } 137 | } 138 | 139 | stage "coverage_raw" { 140 | depends_on = [stage.tests] 141 | script = "go tool covdata percent -i=${local.coverage_data_dir}" 142 | } 143 | stage "coverage_merge" { 144 | depends_on = [stage.coverage_raw, stage.coverage_unit_tests] 145 | script = "go tool covdata merge -i=${local.coverage_data_dir},${local.coverage_data_interactive_dir} -o=${local.coverage_merge_dir}" 146 | } 147 | stage "coverage" { 148 | depends_on = [stage.coverage_merge] 149 | script = "go tool covdata textfmt -i=${local.coverage_merge_dir} -o=coverage.out" 150 | } 151 | stage "coverage_unit_tests" { 152 | depends_on = [stage.build] 153 | dir = ".." 154 | script = "go test ./... -coverprofile=coverage_unit_tests.out" 155 | env { 156 | name = "PROMPT_GOCOVERDIR" 157 | value = local.coverage_data_interactive_dir 158 | } 159 | } 160 | 161 | -------------------------------------------------------------------------------- /togomak.hcl: -------------------------------------------------------------------------------- 1 | togomak { 2 | version = 2 3 | } 4 | 5 | 6 | stage "fmt" { 7 | script = "go fmt github.com/srevinsaju/togomak/v1/..." 8 | lifecycle { 9 | phase = ["default", "validate"] 10 | } 11 | } 12 | 13 | stage "vet" { 14 | script = "go vet github.com/srevinsaju/togomak/v1/..." 15 | lifecycle { 16 | phase = ["default", "validate"] 17 | } 18 | } 19 | 20 | stage "build" { 21 | depends_on = [stage.fmt, stage.vet] 22 | script = "go build -v -o ./cmd/togomak/togomak github.com/srevinsaju/togomak/v1/cmd/togomak" 23 | lifecycle { 24 | phase = ["default", "build"] 25 | } 26 | } 27 | 28 | stage "install" { 29 | depends_on = [stage.build] 30 | script = "go install github.com/srevinsaju/togomak/v1/cmd/togomak" 31 | lifecycle { 32 | phase = ["default", "install"] 33 | } 34 | } 35 | 36 | stage "docs_serve" { 37 | daemon { 38 | enabled = true 39 | } 40 | if = false 41 | script = "cd docs && mdbook serve" 42 | } 43 | --------------------------------------------------------------------------------