├── .github ├── FUNDING.yml └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── LICENSE ├── Makefile ├── README.md ├── docs ├── proposals-declined │ ├── README │ ├── boilerplate │ │ ├── README.md │ │ ├── TODO │ │ └── boilerplate.hcl │ └── namespace │ │ ├── README.md │ │ ├── TODO │ │ └── namespace.hcl └── proposals │ ├── stream │ ├── TODO │ └── stream.variant │ └── testmock │ ├── TODO │ ├── manifests │ └── .gitkeep │ ├── simple.variant │ └── simple_test.variant ├── examples ├── advanced │ ├── dynamic-config-inheritance │ │ ├── config │ │ │ ├── globals.yaml │ │ │ ├── ue2-globals.yaml │ │ │ ├── ue2-prod.yaml │ │ │ └── ue3-prod.yaml │ │ ├── main.variant │ │ ├── main_test.variant │ │ └── userfunc.variant │ ├── import-multi │ │ ├── baz │ │ │ ├── baz.variant │ │ │ └── redundant │ │ │ │ └── .gitkeep │ │ ├── export.sh │ │ ├── import.variant │ │ └── import_test.variant │ ├── import-remote │ │ ├── import.variant │ │ └── import_test.variant │ ├── import │ │ ├── foo │ │ │ └── foo.variant │ │ ├── import.variant │ │ ├── import_test.variant │ │ └── mycli │ ├── logcollection │ │ ├── .gitignore │ │ ├── logcollection.variant │ │ └── logcollection_test.variant │ ├── nested-import-global-propagation-default │ │ ├── lib │ │ │ └── lib.variant │ │ └── main.variant │ ├── nested-import-global-propagation-incompatible-type │ │ ├── lib │ │ │ └── lib.variant │ │ └── main.variant │ ├── nested-import-global-propagation │ │ ├── defaultdir │ │ │ └── project │ │ ├── lib │ │ │ └── lib.variant │ │ ├── main.variant │ │ └── overridedir │ │ │ └── project │ ├── source │ │ ├── README.md │ │ ├── crds │ │ │ ├── source.toolkit.fluxcd.io_buckets.yaml │ │ │ ├── source.toolkit.fluxcd.io_gitrepositories.yaml │ │ │ ├── source.toolkit.fluxcd.io_helmcharts.yaml │ │ │ └── source.toolkit.fluxcd.io_helmrepositories.yaml │ │ ├── main.variant │ │ └── sources │ │ │ └── source.yaml │ ├── terraform-and-helmfile-wrapper │ │ ├── example.variant │ │ ├── example_test.variant │ │ ├── helmfiles │ │ │ └── a │ │ │ │ └── helmfile.yaml │ │ └── terraform │ │ │ └── b │ │ │ └── main.tf │ └── userfunc-local-scope │ │ ├── config │ │ ├── globals.yaml │ │ ├── ue2-globals.yaml │ │ ├── ue2-prod.yaml │ │ └── ue3-prod.yaml │ │ ├── main.variant │ │ └── main_test.variant ├── complex │ ├── complex.variant │ ├── complex_bar_baz.variant │ └── complex_cmd1.variant ├── concurrency │ ├── concurrency.variant │ └── concurrency_test.variant ├── conditional_run │ ├── example.variant │ └── example_test.variant ├── config │ ├── app1 │ │ ├── default.yml │ │ └── prod.yaml │ ├── conf │ │ ├── app1.prod.yaml │ │ ├── app1.yaml │ │ ├── app2.yaml │ │ ├── defaults.yaml │ │ ├── dev.yaml │ │ └── prod.yaml │ ├── config.variant │ └── config_test.variant ├── controller │ ├── README.md │ ├── main.variant │ ├── reconcilation.yaml │ ├── resource.1.yaml │ ├── resource.2.yaml │ └── variant.crds.yaml ├── defaults │ ├── defaults.variant │ └── defaults_test.variant ├── depends_on │ ├── depends_on.variant │ └── depends_on_test.variant ├── exec │ ├── example.variant │ └── example_test.variant ├── getting-started │ ├── .gitignore │ └── getting-started.variant ├── globals │ ├── example.variant │ └── example_test.variant ├── issues │ ├── 8-logging │ │ └── example.variant │ ├── 9-interactive │ │ └── interactive.variant │ ├── cant-convert-go-str-to-bool │ │ ├── conf.yaml │ │ ├── example.variant │ │ └── example_test.variant │ └── sweetops-CFFQ9GFB5-p1586798062189700 │ │ ├── projects │ │ └── helmfiles │ │ │ └── myproj │ │ │ └── helmfile.yaml │ │ └── shell.variant ├── k8s │ ├── default.variantmod │ ├── default.variantmod.lock │ └── module.variant ├── module │ ├── Dockerfile │ ├── Dockerfile.tpl │ ├── default.variantmod │ ├── default.variantmod.lock │ ├── module.variant │ └── module_test.variant ├── options-json │ ├── options.variant.json │ └── options_test.variant ├── options │ ├── options.variant │ └── options_test.variant ├── rubyrunner │ └── rubyrunner.variant ├── secret │ ├── sec │ │ ├── app1.prod.yaml │ │ ├── app1.yaml │ │ ├── app2.yaml │ │ └── prod.yaml │ ├── secret.variant │ └── secret_test.variant ├── simple │ ├── manifests │ │ └── .gitkeep │ ├── mocks │ │ └── kubectl │ │ │ ├── kubectl │ │ │ └── kubectl.variant │ ├── simple.variant │ ├── simple_test.variant │ └── var_types.variant ├── testing-with-expectations │ ├── example.variant │ └── example_test.variant ├── testing │ ├── example.variant │ └── example_test.variant └── variables │ ├── example.variant │ └── example_test.variant ├── export_binary.go ├── export_go.go ├── foo.json ├── foo.txt ├── go.mod ├── go.sum ├── hack ├── print-replaces.go └── sdk-vars.sh ├── main.go ├── main_test.go ├── pipe.go ├── pkg ├── app │ ├── app.go │ ├── app_log.go │ ├── app_shim.go │ ├── app_test.go │ ├── config_sources.go │ ├── cty2go.go │ ├── export.go │ ├── go_to_cty.go │ ├── job.go │ ├── load.go │ ├── load_test.go │ ├── log.go │ ├── run.go │ ├── run_args.go │ ├── source.go │ ├── survey.go │ ├── testdeps_deps.go │ ├── testing_match.go │ ├── testlog_log.go │ ├── trace.go │ ├── types.go │ └── values.go ├── cmd │ └── main.go ├── conf │ ├── funcs.go │ ├── jsonpath.go │ └── load.go ├── controller │ ├── api.go │ ├── capture.go │ ├── config.go │ ├── controller.go │ ├── handler.go │ └── run.go ├── fs │ └── fs.go ├── kube │ ├── client.go │ └── restconfig.go ├── sdk │ └── vars.go ├── slack │ ├── README.md │ ├── http.go │ ├── interaction.go │ ├── slack.go │ └── slashcommand.go └── source │ └── src.go ├── slack.go ├── test ├── export │ └── simple │ │ ├── dst │ │ └── .gitignore │ │ └── src │ │ └── src.variant ├── shebang │ └── myapp │ │ ├── myapp │ │ └── myapp.variant └── variant_test.go ├── types.go └── variant.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: 2 | - mumoshu 3 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: [push] 3 | jobs: 4 | test: 5 | runs-on: ubuntu-latest 6 | strategy: 7 | matrix: 8 | go: 9 | - 1.16.x 10 | name: Go ${{ matrix.go }} test 11 | steps: 12 | - uses: actions/checkout@master 13 | - name: Setup Go 14 | uses: actions/setup-go@v1 15 | with: 16 | go-version: ${{ matrix.go }} 17 | - name: Run go mod download 18 | run: go mod download 19 | - name: Install SSH key 20 | uses: shimataro/ssh-key-action@v2 21 | with: 22 | key: ${{ secrets.SSH_KEY }} 23 | known_hosts: ${{ secrets.KNOWN_HOSTS }} 24 | - name: Run tests 25 | run: | 26 | which kubectl 27 | sudo apt-get update -y 28 | sudo apt-get install ruby -y 29 | GITHUB_REF=refs/heads/v0.0.0 make test 30 | smoke: 31 | runs-on: ubuntu-latest 32 | strategy: 33 | matrix: 34 | go: 35 | - 1.16.x 36 | name: Go ${{ matrix.go }} smoke test 37 | steps: 38 | - uses: actions/checkout@master 39 | - name: Setup Go 40 | uses: actions/setup-go@v1 41 | with: 42 | go-version: ${{ matrix.go }} 43 | - name: Run go mod download 44 | run: go mod download 45 | - name: Install SSH key 46 | uses: shimataro/ssh-key-action@v2 47 | with: 48 | key: ${{ secrets.SSH_KEY }} 49 | known_hosts: ${{ secrets.KNOWN_HOSTS }} 50 | - name: Run tests 51 | run: | 52 | GITHUB_REF=refs/heads/v0.0.0 make smoke 53 | lint: 54 | runs-on: ubuntu-latest 55 | strategy: 56 | matrix: 57 | go: 58 | - 1.16.x 59 | name: Go ${{ matrix.go }} lint 60 | steps: 61 | - uses: actions/checkout@master 62 | - name: Setup Go 63 | uses: actions/setup-go@v1 64 | with: 65 | go-version: ${{ matrix.go }} 66 | - name: Run go mod download 67 | run: go mod download 68 | - name: Run golangci-lint 69 | run: make lint 70 | goreleaser-test: 71 | runs-on: ubuntu-latest 72 | steps: 73 | - 74 | name: Checkout 75 | uses: actions/checkout@v1 76 | - 77 | name: Set up Go 78 | uses: actions/setup-go@v1 79 | with: 80 | go-version: 1.16.x 81 | - 82 | name: Set goreleaser .Env 83 | run: | 84 | GITHUB_REF=refs/heads/v0.0.0 hack/sdk-vars.sh 85 | - 86 | name: Run GoReleaser 87 | uses: goreleaser/goreleaser-action@v1 88 | with: 89 | version: latest 90 | args: release --rm-dist --skip-publish 91 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | jobs: 9 | goreleaser: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - 13 | name: Checkout 14 | uses: actions/checkout@v1 15 | - 16 | name: Set up Go 17 | uses: actions/setup-go@v1 18 | with: 19 | go-version: 1.16.x 20 | - 21 | name: Set goreleaser .Env 22 | run: | 23 | hack/sdk-vars.sh 24 | - 25 | name: Run GoReleaser 26 | uses: goreleaser/goreleaser-action@v1 27 | with: 28 | version: latest 29 | args: release --rm-dist 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *~ 3 | examples/module/.variant 4 | variant 5 | /bin/* 6 | \#*# 7 | .variant2 8 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | gci: 3 | # put imports beginning with prefix after 3rd-party packages; 4 | # only support one prefix 5 | # if not set, use goimports.local-prefixes 6 | local-prefixes: github.com/mumoshu/variant2 7 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: variant 2 | builds: 3 | - main: ./pkg/cmd 4 | env: 5 | - CGO_ENABLED=0 6 | ldflags: 7 | - -s -w -X github.com/mumoshu/variant2/Version={{.Version}} 8 | - -X github.com/mumoshu/variant2/pkg/sdk.Version={{.Env.VERSION}} 9 | - -X github.com/mumoshu/variant2/pkg/sdk.ModReplaces={{.Env.MOD_REPLACES}} 10 | changelog: 11 | filters: 12 | # commit messages matching the regexp listed here will be removed from 13 | # the changelog 14 | # Default is empty 15 | exclude: 16 | - '^docs:' 17 | - typo 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Yusuke Kuoka 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VARIANT_SDK = github.com/mumoshu/variant2/pkg/sdk 2 | 3 | # NOTE: 4 | # You can test the versioned build with e.g. `GITHUB_REF=refs/heads/v0.36.0 make build` 5 | .PHONY: build 6 | build: 7 | @echo "Building variant" 8 | @{ \ 9 | set -e ;\ 10 | . hack/sdk-vars.sh ;\ 11 | echo Using $(VAARIANT_SDK).Version=$${VERSION} ;\ 12 | echo Using $(VAARIANT_SDK).ModReplaces=$${MOD_REPLACES} ;\ 13 | set -x ;\ 14 | go build \ 15 | -ldflags "-X $(VARIANT_SDK).Version=$${VERSION} -X $(VARIANT_SDK).ModReplaces=$${MOD_REPLACES}" \ 16 | -o variant ./pkg/cmd ;\ 17 | } 18 | 19 | bin/goimports: 20 | echo "Installing goimports" 21 | @{ \ 22 | set -e ;\ 23 | INSTALL_TMP_DIR=$$(mktemp -d) ;\ 24 | cd $$INSTALL_TMP_DIR ;\ 25 | go mod init tmp ;\ 26 | GOBIN=$(PWD)/bin go install golang.org/x/tools/cmd/goimports ;\ 27 | rm -rf $$INSTALL_TMP_DIR ;\ 28 | } 29 | 30 | bin/source-controller: 31 | echo "Installing source-controller" 32 | @{ \ 33 | set -e ;\ 34 | INSTALL_TMP_DIR=$$(mktemp -d) ;\ 35 | cd $$INSTALL_TMP_DIR ;\ 36 | go mod init tmp ;\ 37 | GOBIN=$(PWD)/bin go install github.com/fluxcd/source-controller ;\ 38 | rm -rf $$INSTALL_TMP_DIR ;\ 39 | } 40 | 41 | bin/gofumpt: 42 | echo "Installing gofumpt" 43 | @{ \ 44 | set -e ;\ 45 | INSTALL_TMP_DIR=$$(mktemp -d) ;\ 46 | cd $$INSTALL_TMP_DIR ;\ 47 | go mod init tmp ;\ 48 | GOBIN=$(PWD)/bin go install mvdan.cc/gofumpt ;\ 49 | rm -rf $$INSTALL_TMP_DIR ;\ 50 | } 51 | 52 | bin/gci: 53 | echo "Installing gci" 54 | @{ \ 55 | set -e ;\ 56 | INSTALL_TMP_DIR=$$(mktemp -d) ;\ 57 | cd $$INSTALL_TMP_DIR ;\ 58 | go mod init tmp ;\ 59 | GOBIN=$(PWD)/bin go install github.com/daixiang0/gci ;\ 60 | rm -rf $$INSTALL_TMP_DIR ;\ 61 | } 62 | 63 | .PHONY: source-controller-crds 64 | source-controller-crds: 65 | # See https://stackoverflow.com/questions/600079/git-how-do-i-clone-a-subdirectory-only-of-a-git-repository/52269934#52269934 66 | echo "Fetching source-controller crds" 67 | @{ \ 68 | set -xe ;\ 69 | INSTALL_TMP_DIR=$$(mktemp -d) ;\ 70 | cd $$INSTALL_TMP_DIR ;\ 71 | git clone \ 72 | --depth 1 \ 73 | --filter=blob:none \ 74 | --no-checkout \ 75 | https://github.com/fluxcd/source-controller ;\ 76 | cd source-controller ;\ 77 | git checkout main -- config/crd/bases ;\ 78 | rm -rf .git ;\ 79 | cp config/crd/bases/* $(PWD)/examples/advanced/source/crds ;\ 80 | rm -rf $$INSTALL_TMP_DIR ;\ 81 | } 82 | 83 | .PHONY: fmt 84 | fmt: bin/goimports bin/gci bin/gofumpt 85 | gofmt -w -s pkg . 86 | bin/gofumpt -w . || : 87 | bin/gci -w -local github.com/mumoshu/variant2 . || : 88 | 89 | 90 | .PHONY: test 91 | test: build 92 | go vet ./... 93 | PATH=$(PWD):$(PATH) go test -race -v ./... 94 | 95 | bin/golangci-lint: 96 | curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.33.0 97 | 98 | .PHONY: lint 99 | lint: bin/golangci-lint 100 | bin/golangci-lint run --tests ./... \ 101 | --timeout 5m \ 102 | --enable-all \ 103 | --disable gochecknoglobals \ 104 | --disable gochecknoinits \ 105 | --disable gomnd,funlen,prealloc,gocritic,lll,gocognit \ 106 | --disable testpackage,goerr113,exhaustivestruct,wrapcheck,paralleltest 107 | 108 | .PHONY: smoke 109 | smoke: export GOBIN=$(shell pwd)/tools 110 | smoke: build 111 | go get github.com/rakyll/statik 112 | 113 | make build 114 | rm -rf build/simple 115 | VARIANT_BUILD_VARIANT_REPLACE=$(shell pwd) \ 116 | PATH=${PATH}:$(GOBIN) \ 117 | ./variant export go examples/simple build/simple 118 | cd build/simple; go build -o simple ./ 119 | build/simple/simple -h | tee smoke.log 120 | grep "Namespace to interact with" smoke.log 121 | 122 | rm build/simple/simple 123 | VARIANT_BUILD_VARIANT_REPLACE=$(shell pwd) \ 124 | PATH=${PATH}:$(GOBIN) \ 125 | ./variant export binary examples/simple build/simple 126 | build/simple/simple -h | tee smoke2.log 127 | grep "Namespace to interact with" smoke2.log 128 | 129 | rm -rf build/import-multi 130 | VARIANT_BUILD_VER=v0.0.0 \ 131 | VARIANT_BUILD_VARIANT_REPLACE=$(shell pwd) \ 132 | PATH=${PATH}:$(GOBIN) \ 133 | ./variant export binary examples/advanced/import-multi build/import-multi 134 | build/import-multi foo baz HELLO > build/import-multi.log 135 | bash -c 'diff <(echo HELLO) <(cat build/import-multi.log)' 136 | 137 | rm build/import-multi.log 138 | cd build && \ 139 | ./import-multi foo baz HELLO > import-multi.log && \ 140 | bash -c 'diff <(echo HELLO) <(cat import-multi.log)' 141 | # Remote imports are cached and embedded into the binary so it shouldn't be fetched/persisted at run time 142 | [ ! -e build/.variant2/cache ] || bash -c 'echo build/.variant2/cache check failed; exit 1' 143 | -------------------------------------------------------------------------------- /docs/proposals-declined/README: -------------------------------------------------------------------------------- 1 | `proposals-declined` contains the collection of proposed but declined proposals to Variant 2. 2 | 3 | Please see each sub-directory for more information including the proposed usage and the alternative. 4 | -------------------------------------------------------------------------------- /docs/proposals-declined/boilerplate/README.md: -------------------------------------------------------------------------------- 1 | `boilerplate` was intended for reducing code repetition across jobs. 2 | 3 | 4 | However it turned out that splitting variant command and using top-level `options` and `variables` to share the common parts seemed better interms of testability. 5 | 6 | Proposed but declined usage: 7 | 8 | ``` 9 | boilerplate "x" { 10 | option "common1" { 11 | 12 | } 13 | } 14 | 15 | job "a" { 16 | boilerplate = "x" 17 | } 18 | 19 | job "b" { 20 | boilerplate = "x" 21 | } 22 | 23 | boilerplate "y" { 24 | option "common2" { 25 | 26 | } 27 | } 28 | 29 | job "c" { 30 | boilerplate = "y" 31 | } 32 | ``` 33 | 34 | Alternative: 35 | 36 | ``` 37 | # cmd1/cmd1.variant 38 | 39 | option "common1" { 40 | 41 | } 42 | 43 | job "a" { 44 | 45 | } 46 | 47 | job "b" { 48 | 49 | } 50 | 51 | # cmd2/cmd2.variant 52 | 53 | option "common2" { 54 | 55 | } 56 | 57 | job "c" { 58 | 59 | } 60 | ``` 61 | -------------------------------------------------------------------------------- /docs/proposals-declined/boilerplate/TODO: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mumoshu/variant2/1c4392339a51a736518ea810cafc7855de82956c/docs/proposals-declined/boilerplate/TODO -------------------------------------------------------------------------------- /docs/proposals-declined/boilerplate/boilerplate.hcl: -------------------------------------------------------------------------------- 1 | boilerplate "commons" { 2 | parameter "param1" { 3 | type = string 4 | } 5 | 6 | option "opt1" { 7 | type = string 8 | } 9 | 10 | variable "var1" { 11 | value = "${param.param1} + ${opt.opt1}" 12 | } 13 | } 14 | 15 | job "test" { 16 | // This is replaced to parameters, options and variables defined in the named boilerplate 17 | boilerplate = "commons" 18 | 19 | exec { 20 | command = "echo" 21 | // So that we can refer to var1 defined in the boilerplate here 22 | args = [var.var1] 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /docs/proposals-declined/namespace/README.md: -------------------------------------------------------------------------------- 1 | `namespace` was intended for reducing code repetition per group of jobs. 2 | 3 | However it turned out that splitting variant command and using top-level `options` and `variables` to share the common parts seemed better in terms of testability. 4 | 5 | Proposed but declined usage: 6 | 7 | ``` 8 | namespace "x" { 9 | option "common1" { 10 | 11 | } 12 | } 13 | 14 | job "x a" { 15 | 16 | } 17 | 18 | job "x b" { 19 | 20 | } 21 | 22 | namespace "y" { 23 | option "common2" { 24 | 25 | } 26 | } 27 | 28 | job "y c" { 29 | 30 | } 31 | ``` 32 | 33 | Alternative: 34 | 35 | ``` 36 | # cmd1/cmd1.variant 37 | 38 | option "common1" { 39 | 40 | } 41 | 42 | job "a" { 43 | 44 | } 45 | 46 | job "b" { 47 | 48 | } 49 | 50 | # cmd2/cmd2.variant 51 | 52 | option "common2" { 53 | 54 | } 55 | 56 | job "c" { 57 | 58 | } 59 | ``` 60 | -------------------------------------------------------------------------------- /docs/proposals-declined/namespace/TODO: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mumoshu/variant2/1c4392339a51a736518ea810cafc7855de82956c/docs/proposals-declined/namespace/TODO -------------------------------------------------------------------------------- /docs/proposals-declined/namespace/namespace.hcl: -------------------------------------------------------------------------------- 1 | namespace "foo" { 2 | # `variant run foo` shows this description 3 | description = "collection of jobs related to foo" 4 | 5 | # `variant run foo -h` shows opt1 as a global flag 6 | option "opt1" { 7 | type = string 8 | } 9 | } 10 | 11 | namespace "foo bar" { 12 | # `variant run foo` shows this description 13 | description = "collection of jobs related to bar" 14 | 15 | # `variant run foo bar -h` shows opt2 as a global flag 16 | option "opt2" { 17 | type = string 18 | } 19 | } 20 | 21 | # opt1 and opt2 are available as persistent flags 22 | # i.e. `variant run foo bar test -h` shows opt1 and opt2 as global flags 23 | job "foo bar test" { 24 | exec { 25 | command = "echo" 26 | args = "opt1=${opt.opt1},opt2=${opt.opt2}" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /docs/proposals/stream/TODO: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mumoshu/variant2/1c4392339a51a736518ea810cafc7855de82956c/docs/proposals/stream/TODO -------------------------------------------------------------------------------- /docs/proposals/stream/stream.variant: -------------------------------------------------------------------------------- 1 | job "test" { 2 | stream { 3 | source "foo" { 4 | # setting `file = "-"` tells Variant to fallback to stdin if the file attr is omitted or the file is not found 5 | file = "foo.txt" 6 | } 7 | 8 | source "bar" { 9 | file = "bar" 10 | 11 | # `poll_on_demand` denotes an infinite source that doesn't trigger update on its own 12 | # with `cache_for = 10s` it fetches from the source only when it passed 10s after the last fetch. 13 | # 14 | # # defauls to invalidate_after = 0s which 15 | # repeat_every = 0s 16 | # invalidate_after = 10s 17 | # #strategy = poll_on_demand 18 | # read = "all" 19 | # emit = false 20 | # 21 | # `periodic` denotes an infinite source that produces data for every interval passed 22 | # 23 | # repeat_every = 1s 24 | # #strategy = periodic 25 | # read = "all" 26 | # line_rate = inf 27 | # emit = true 28 | # 29 | # 30 | # `once` denotes an (sometimes finite or possibly infinite) source that produces data from a file or a job run. 31 | # If it was a file and the file reader reached to EOF, the stream is closed hence it was a finite source. 32 | # If it was a job run and the job finished running, the stream is closed hence it was a finite source. 33 | # If it was a job run and the job kept running forever, the stream is never closed hence it was an infinite source. 34 | # 35 | # #repeat_every = 0s 36 | # #strategy = "once" 37 | # read = "line" 38 | # emit = true 39 | # 40 | # reads up to 10 lines per second 41 | # #line_rate = 1 * second / 10 42 | # read_every = 0.1s 43 | } 44 | 45 | source "baz" { 46 | file = "baz.txt" 47 | # setting `file = ""` results in the job `baz` being run by Variant 48 | job = { 49 | name = "baz" 50 | args = { 51 | } 52 | } 53 | 54 | # retries up to 3 times on failure 55 | retries = 3 56 | # exponential backoff 57 | backoff = (2 * seconds * pow(retry.number, 2)) 58 | } 59 | 60 | # "latest" is the default and the only strategy at the moment that runs the job each time new input is arrived to any of the sources 61 | #merge_strategy = "latest" 62 | 63 | # Runs the job `print` up to 2 times per 0.1 second 64 | # See https://github.com/golang/go/wiki/RateLimiting 65 | rate_limit = 1 * second / 10 66 | burst_limit = 2 67 | 68 | # Stops Variant with non-zero exit code after 10 seconds with no input frmo any source 69 | timeout = 10 * seconds 70 | 71 | # errors from `print` job doesn't stop this stream. set `until` to stop conditionally. 72 | flow { 73 | run "print" { 74 | text = "foo=${source.foo.item}, bar=${source.bar.item}, baz=${source.baz.item}" 75 | } 76 | } 77 | 78 | # Set `suceed_on = run.err == ""` to stop this stream with a successful exit code(=0) on the first successful run of the job `print` 79 | #suceed_on = source.foo.closed && source.bar.closed && source.baz.closed 80 | 81 | # source.*.error contains non-empty message string of the last error occurred for the source 82 | #fail_on = source.foo.error != "" || source.bar.error != "" || source.baz.error != "" 83 | } 84 | } 85 | 86 | job "print" { 87 | parameter "text" { 88 | type = string 89 | } 90 | 91 | exec { 92 | command = "echo" 93 | args = [param.text] 94 | } 95 | } 96 | 97 | job "baz" { 98 | parameter "text" { 99 | type = string 100 | } 101 | 102 | exec { 103 | command = "bash" 104 | args = ["-c", "while sleep 1; do echo ${param.text}; done"] 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /docs/proposals/testmock/TODO: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mumoshu/variant2/1c4392339a51a736518ea810cafc7855de82956c/docs/proposals/testmock/TODO -------------------------------------------------------------------------------- /docs/proposals/testmock/manifests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mumoshu/variant2/1c4392339a51a736518ea810cafc7855de82956c/docs/proposals/testmock/manifests/.gitkeep -------------------------------------------------------------------------------- /docs/proposals/testmock/simple.variant: -------------------------------------------------------------------------------- 1 | option namespace { 2 | description = "Namespace to interact with" 3 | type = string 4 | default = "default" 5 | short = "n" 6 | } 7 | 8 | job "shell" { 9 | parameter "script" { 10 | type = string 11 | } 12 | 13 | parameter "path" { 14 | type = string 15 | } 16 | 17 | exec { 18 | command = "bash" 19 | args = ["-c", param.script] 20 | env = { 21 | PATH = param.path 22 | } 23 | } 24 | } 25 | 26 | job "app deploy" { 27 | option "path" { 28 | type = string 29 | default = ".:${abspath("${context.sourcedir}/mocks/kubectl")}:/bin:/usr/bin" 30 | } 31 | run "shell" { 32 | script = <_JOB_ON_APPLY` to the job ran on resource creation or update, 36 | and `_JOB_ON_DESTROY` to the job ran on resource deletion. The only remaining required envvar is `VARIANT_CONTROLLER_NAME`, 37 | which must be set to whatever name that the controller uses as the name of itself. 38 | 39 | As we've seen in the example in the beginning of this section, the on-apply job is `apply` and the on-destroy job is `destroy` so that `variant` invocation 40 | should look like: 41 | 42 | ```console 43 | $ VARIANT_CONTROLLER_JOB_ON_APPLY=apply \ 44 | VARIANT_CONTROLLER_JOB_ON_DESTROY=destroy \ 45 | VARIANT_CONTROLLER_NAME=resource \ 46 | variant 47 | ``` 48 | 49 | `variant` uses `core.variant.run/v1beta1` `Resource` as the resource to be reconciled by the controller. 50 | 51 | That being said, you can let the controller reconcile your resource by creating a `Resource` object with correct arguments - 52 | `env` and `ref` in this example - under the object's `spec` field: 53 | 54 | ```console 55 | $ kubectl apply -f <(cat EOS 56 | apiVersion: core.variant.run/v1beta1 57 | kind: Resource 58 | metadata: 59 | name: myresource 60 | spec: 61 | env: preview 62 | ref: abc1234 63 | EOS 64 | ) 65 | ``` 66 | 67 | Within a few seconds, the controller will reconcile your `Resource` by running `variant run apply --env preview --ref abc1234`. 68 | 69 | You can verify that by tailing controller logs by `kubectl logs`, or browsing the `Reconcilation` object that is created by 70 | the controller to record the reconciliation details: 71 | 72 | ```console 73 | $ kubectl get reconciliation 74 | NAME AGE 75 | myresource-2 12m 76 | ``` 77 | 78 | ```console 79 | $ kubectl get -o yaml reconciliation myresource-2 80 | apiVersion: core.variant.run/v1beta1 81 | kind: Reconciliation 82 | metadata: 83 | creationTimestamp: "2020-10-28T12:05:55Z" 84 | generation: 1 85 | labels: 86 | core.variant.run/controller: resource 87 | core.variant.run/event: apply 88 | core.variant.run/pod: YOUR_HOST_OR_POD_NAME 89 | name: myresource-2 90 | namespace: default 91 | spec: 92 | combinedLogs: 93 | data: | 94 | Deploying abc2345 to preview 95 | job: apply 96 | resource: 97 | env: preview 98 | ref: abc2345 99 | ``` 100 | 101 | Updating the `Resource` object will result in `variant` running `variant run apply` with the updated arguments: 102 | 103 | ```console 104 | $ kubectl apply -f <(cat <_FOR_API_VERSION`, 201 | and different kind than the default `Resource` by setting `_FOR_KIND`. 202 | 203 | For example, to let the controller watch and reconcile `whatever.example.com/v1alpha1` `MyCustomStack` objects, run `variant` 204 | like: 205 | 206 | ```console 207 | $ VARIANT_CONTROLLER_JOB_ON_APPLY=apply \ 208 | VARIANT_CONTROLLER_JOB_ON_DESTROY=destroy \ 209 | VARIANT_CONTROLLER_NAME=my-custom-stack \ 210 | VARIANT_CONTROLLER_FOR_API_VERSION=whatever.example.com/v1alpha1 \ 211 | VARIANT_CONTROLLER_FOR_KIND=MyCustomStack \ 212 | variant 213 | ``` 214 | -------------------------------------------------------------------------------- /examples/controller/main.variant: -------------------------------------------------------------------------------- 1 | option "env" { 2 | type = string 3 | } 4 | 5 | option "ref" { 6 | type = string 7 | } 8 | 9 | job "apply" { 10 | exec { 11 | command = "echo" 12 | args = ["Deploying ${opt.ref} to ${opt.env}"] 13 | } 14 | } 15 | 16 | job "destroy" { 17 | exec { 18 | command = "echo" 19 | args = ["Destroying ${opt.env}"] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /examples/controller/reconcilation.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: core.variant.run/v1beta1 2 | kind: Reconciliation 3 | metadata: 4 | name: myresource-abc 5 | spec: 6 | job: "apply" 7 | resource: 8 | env: preview 9 | ref: abc2345 10 | -------------------------------------------------------------------------------- /examples/controller/resource.1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: core.variant.run/v1beta1 2 | kind: Resource 3 | metadata: 4 | name: myresource 5 | spec: 6 | env: preview 7 | ref: abc1234 8 | -------------------------------------------------------------------------------- /examples/controller/resource.2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: core.variant.run/v1beta1 2 | kind: Resource 3 | metadata: 4 | name: myresource 5 | spec: 6 | env: preview 7 | ref: abc2345 8 | -------------------------------------------------------------------------------- /examples/controller/variant.crds.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1beta1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | name: resources.core.variant.run 5 | spec: 6 | group: core.variant.run 7 | versions: 8 | - name: v1beta1 9 | served: true 10 | storage: true 11 | names: 12 | kind: Resource 13 | plural: resources 14 | singular: resource 15 | scope: Namespaced 16 | --- 17 | apiVersion: apiextensions.k8s.io/v1beta1 18 | kind: CustomResourceDefinition 19 | metadata: 20 | name: reconciliations.core.variant.run 21 | spec: 22 | group: core.variant.run 23 | versions: 24 | - name: v1beta1 25 | served: true 26 | storage: true 27 | names: 28 | kind: Reconciliation 29 | plural: reconciliations 30 | singular: reconciliation 31 | scope: Namespaced 32 | -------------------------------------------------------------------------------- /examples/defaults/defaults.variant: -------------------------------------------------------------------------------- 1 | option "somemap_deprecated" { 2 | type = map(string) 3 | default = map("foo", "FOO", "bar", "BAR") 4 | } 5 | 6 | option "somemap" { 7 | type = map(string) 8 | default = {foo="FOO", bar="BAR"} 9 | } 10 | 11 | option "emptymap" { 12 | type = map(string) 13 | # map() is invalid. 14 | # it results in this error: `Error in function call; Call to function "map" failed: map requires an even number of two or more arguments, got 0.` 15 | #default = map() 16 | default = {} 17 | } 18 | 19 | option "sometuple" { 20 | type = tuple([string,number,bool]) 21 | default = ["x", 1, true] 22 | } 23 | 24 | option "someobj" { 25 | type = object({foo=string, bar=number, baz=bool}) 26 | default = {foo="FOO", bar=1, baz=true} 27 | } 28 | 29 | job "example" { 30 | exec { 31 | command = "bash" 32 | args = [ 33 | "-c", < github.com/mumoshu/whitebox-controller v0.5.1-0.20201028130131-ac7a0743254b 56 | 57 | // Required to fix go mod issue that k8s.io/client-go is somehow "updated" to invalid "v10.0.0+incompatible" on build 58 | replace k8s.io/client-go v10.0.0+incompatible => k8s.io/client-go v0.18.9 59 | -------------------------------------------------------------------------------- /hack/print-replaces.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func main() { 11 | content, err := ioutil.ReadFile("go.mod") 12 | if err != nil { 13 | fmt.Fprintf(os.Stderr, "Error: %v\n", err) 14 | os.Exit(1) 15 | } 16 | 17 | var replaces []string 18 | 19 | for _, l := range strings.Split(string(content), "\n") { 20 | if !strings.HasPrefix(l, "replace ") { 21 | continue 22 | } 23 | 24 | if !strings.Contains(l, " => ") { 25 | fmt.Fprintf(os.Stderr, "Unexpected line: ` => ` expected: %s\n", l) 26 | os.Exit(1) 27 | } 28 | 29 | l = strings.ReplaceAll(l, "replace ", "") 30 | l = strings.ReplaceAll(l, " => ", "=") 31 | l = strings.ReplaceAll(l, " ", "@") 32 | l = strings.TrimRight(l, "\n") 33 | 34 | replaces = append(replaces, l) 35 | } 36 | 37 | fmt.Print(strings.Join(replaces, ",")) 38 | } 39 | -------------------------------------------------------------------------------- /hack/sdk-vars.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | tag=${GITHUB_REF##*/} 6 | 7 | if [ -z "${tag}" ]; then 8 | echo GITHUB_REF must be set 1>&2 9 | exit 1 10 | fi 11 | 12 | export VERSION=${tag} 13 | export MOD_REPLACES=$(go run hack/print-replaces.go) 14 | 15 | if [ ! -z "${GITHUB_ENV}" ]; then 16 | echo "VERSION=${VERSION}" >> $GITHUB_ENV 17 | echo "MOD_REPLACES=${MOD_REPLACES}" >> $GITHUB_ENV 18 | fi 19 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package variant 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/mumoshu/variant2/pkg/controller" 8 | ) 9 | 10 | func RunMain(env Env, opts ...Option) error { 11 | cmd, path, args := GetPathAndArgsFromEnv(env) 12 | 13 | m, err := Load(FromPath(path, func(m *Main) { 14 | m.Command = cmd 15 | 16 | for _, o := range opts { 17 | o(m) 18 | } 19 | })) 20 | if err != nil { 21 | return fmt.Errorf("loading command: %w", err) 22 | } 23 | 24 | if controller.RunRequested() { 25 | return controller.Run(func(args []string) (string, error) { 26 | out, err := controller.CaptureOutput(func(stdout, stderr io.Writer) error { 27 | return m.Run(args, RunOptions{ 28 | Stdout: stdout, 29 | Stderr: stdout, 30 | DisableLocking: false, 31 | }) 32 | }) 33 | 34 | //nolint:wrapcheck 35 | return out, err 36 | }) 37 | } 38 | 39 | return m.Run(args, RunOptions{DisableLocking: false}) 40 | } 41 | -------------------------------------------------------------------------------- /pipe.go: -------------------------------------------------------------------------------- 1 | package variant 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "io/ioutil" 8 | ) 9 | 10 | func Pipe() (func() (*bytes.Buffer, error), io.WriteCloser) { 11 | r, w := io.Pipe() 12 | 13 | out := &bytes.Buffer{} 14 | 15 | outDone := make(chan error, 1) 16 | 17 | go func() { 18 | bs, err := ioutil.ReadAll(r) 19 | if err != nil { 20 | outDone <- err 21 | 22 | return 23 | } 24 | 25 | if _, err := out.Write(bs); err != nil { 26 | outDone <- err 27 | 28 | return 29 | } 30 | 31 | outDone <- nil 32 | }() 33 | 34 | return func() (*bytes.Buffer, error) { 35 | err := <-outDone 36 | 37 | if !errors.Is(err, io.EOF) { 38 | return nil, err 39 | } 40 | 41 | return out, nil 42 | }, w 43 | } 44 | -------------------------------------------------------------------------------- /pkg/app/app_log.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | 6 | gohcl2 "github.com/hashicorp/hcl/v2/gohcl" 7 | "github.com/zclconf/go-cty/cty" 8 | ) 9 | 10 | func (app *App) newLogCollector(file string, j JobSpec, jobCtx *JobContext) LogCollector { 11 | logCollector := LogCollector{ 12 | FilePath: file, 13 | CollectFn: func(evt Event) (*string, bool, error) { 14 | condVars := map[string]cty.Value{} 15 | for k, v := range jobCtx.evalContext.Variables { 16 | condVars[k] = v 17 | } 18 | condVars["event"] = evt.toCty() 19 | condCtx := *jobCtx.evalContext 20 | condCtx.Variables = condVars 21 | 22 | for _, c := range j.Log.Collects { 23 | var condVal cty.Value 24 | if diags := gohcl2.DecodeExpression(c.Condition, &condCtx, &condVal); diags.HasErrors() { 25 | return nil, false, diags 26 | } 27 | vv, err := ctyToGo(condVal) 28 | if err != nil { 29 | return nil, false, err 30 | } 31 | 32 | b, ok := vv.(bool) 33 | if !ok { 34 | return nil, false, fmt.Errorf("unexpected type of condition value: want bool, got %T", vv) 35 | } 36 | 37 | if !b { 38 | continue 39 | } 40 | 41 | formatVars := map[string]cty.Value{} 42 | for k, v := range jobCtx.evalContext.Variables { 43 | formatVars[k] = v 44 | } 45 | formatVars["event"] = evt.toCty() 46 | formatCtx := *jobCtx.evalContext 47 | formatCtx.Variables = condVars 48 | 49 | var formatVal cty.Value 50 | if diags := gohcl2.DecodeExpression(c.Format, &formatCtx, &formatVal); diags.HasErrors() { 51 | return nil, false, diags 52 | } 53 | formatV, err := ctyToGo(formatVal) 54 | if err != nil { 55 | return nil, false, err 56 | } 57 | f, ok := formatV.(string) 58 | if !ok { 59 | return nil, false, fmt.Errorf("unexpected type of format value: want string, got %T", f) 60 | } 61 | 62 | return &f, true, nil 63 | } 64 | 65 | return nil, false, nil 66 | }, 67 | ForwardFn: func(log Log) error { 68 | logCty := cty.MapVal(map[string]cty.Value{ 69 | "file": cty.StringVal(log.File), 70 | }) 71 | evalCtx := *jobCtx.evalContext 72 | evalCtx.Variables["log"] = logCty 73 | 74 | newJobCtx := *jobCtx 75 | newJobCtx.evalContext = &evalCtx 76 | 77 | for _, f := range j.Log.Forwards { 78 | _, err := app.dispatchRunJob(nil, &newJobCtx, eitherJobRun{static: f.Run}, false) 79 | if err != nil { 80 | return err 81 | } 82 | } 83 | 84 | return nil 85 | }, 86 | } 87 | 88 | return logCollector 89 | } 90 | -------------------------------------------------------------------------------- /pkg/app/app_test.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestExampleComplex(t *testing.T) { 10 | app, err := New(FromDir("../../examples/complex")) 11 | app.Stdout = os.Stdout 12 | app.Stderr = os.Stderr 13 | 14 | if err != nil { 15 | app.ExitWithError(err) 16 | } 17 | 18 | testcases := []struct { 19 | cmd string 20 | args map[string]interface{} 21 | opts map[string]interface{} 22 | err string 23 | }{ 24 | { 25 | cmd: "", 26 | args: map[string]interface{}{ 27 | "param1": "param1v", 28 | }, 29 | opts: map[string]interface{}{ 30 | "opt1": "opt1", 31 | }, 32 | err: "nothing to run", 33 | }, 34 | { 35 | cmd: "cmd1", 36 | args: map[string]interface{}{ 37 | "param1": "param1v", 38 | "param2": "param2", 39 | "param3": "param3", 40 | }, 41 | opts: map[string]interface{}{ 42 | "opt1": "opt1", 43 | }, 44 | }, 45 | { 46 | cmd: "bar baz", 47 | args: map[string]interface{}{ 48 | "param1": "param1v", 49 | "param2": "param2", 50 | "param3": "param3", 51 | }, 52 | opts: map[string]interface{}{ 53 | "opt1": "opt1", 54 | }, 55 | }, 56 | } 57 | 58 | for i := range testcases { 59 | tc := testcases[i] 60 | 61 | t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { 62 | cmd := tc.cmd 63 | args := tc.args 64 | opts := tc.opts 65 | 66 | _, err := app.Run(cmd, args, opts) 67 | if err != nil { 68 | if tc.err == "" { 69 | t.Errorf("unexpected error: %v", err) 70 | } else if tc.err != err.Error() { 71 | t.Errorf("unexpected error: want %q, got %q", tc.err, err.Error()) 72 | } 73 | } else if tc.err != "" { 74 | t.Errorf("expected error did not occur: want %q, got none", tc.err) 75 | } 76 | }) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /pkg/app/config_sources.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "reflect" 8 | 9 | "github.com/hashicorp/hcl/v2" 10 | gohcl2 "github.com/hashicorp/hcl/v2/gohcl" 11 | "golang.org/x/xerrors" 12 | ) 13 | 14 | type configFragment struct { 15 | data []byte 16 | key string 17 | format string 18 | } 19 | 20 | func loadConfigSourceContent(sourceSpec ConfigSource) (*hcl.BodyContent, error) { 21 | body := sourceSpec.Body 22 | 23 | var val reflect.Value 24 | 25 | switch sourceSpec.Type { 26 | case "file": 27 | rv := reflect.ValueOf(&SourceFile{}) 28 | if rv.Kind() != reflect.Ptr { 29 | panic(fmt.Sprintf("target value must be a pointer, not %s", rv.Type().String())) 30 | } 31 | 32 | val = rv.Elem() 33 | case "job": 34 | rv := reflect.ValueOf(&SourceJob{}) 35 | if rv.Kind() != reflect.Ptr { 36 | panic(fmt.Sprintf("target value must be a pointer, not %s", rv.Type().String())) 37 | } 38 | 39 | val = rv.Elem() 40 | default: 41 | return nil, fmt.Errorf("config source %q is not implemented. It must be either \"file\" or \"job\", so that it looks like `source file {` or `source file {`", sourceSpec.Type) 42 | } 43 | 44 | schema, partial := gohcl2.ImpliedBodySchema(val.Interface()) 45 | 46 | var content *hcl.BodyContent 47 | 48 | var _ hcl.Body 49 | 50 | var diags hcl.Diagnostics 51 | 52 | if partial { 53 | content, _, diags = body.PartialContent(schema) 54 | } else { 55 | content, diags = body.Content(schema) 56 | } 57 | 58 | if content == nil { 59 | return nil, diags 60 | } 61 | 62 | return content, nil 63 | } 64 | 65 | func (app *App) loadConfigSource(jobCtx *JobContext, confCtx *hcl.EvalContext, sourceSpec ConfigSource) ([]configFragment, error) { 66 | var err error 67 | 68 | var fragments []configFragment 69 | 70 | switch sourceSpec.Type { 71 | case "file": 72 | fragments, err = loadFileConfigSource(confCtx, sourceSpec) 73 | if err != nil { 74 | return nil, err 75 | } 76 | case "job": 77 | fragments, err = app.loadJobConfigSource(jobCtx, confCtx, sourceSpec) 78 | if err != nil { 79 | return nil, err 80 | } 81 | default: 82 | return nil, fmt.Errorf("config source %q is not implemented. It must be either \"file\" or \"job\", so that it looks like `source file {` or `source file {`", sourceSpec.Type) 83 | } 84 | 85 | return fragments, nil 86 | } 87 | 88 | func (app *App) loadJobConfigSource(jobCtx *JobContext, confCtx *hcl.EvalContext, sourceSpec ConfigSource) ([]configFragment, error) { 89 | var source SourceJob 90 | if err := gohcl2.DecodeBody(sourceSpec.Body, confCtx, &source); err != nil { 91 | return nil, xerrors.Errorf("decoding job body: %w", err) 92 | } 93 | 94 | args, err := buildArgsFromExpr(jobCtx.WithEvalContext(confCtx).Ptr(), source.Args) 95 | if err != nil { 96 | return nil, err 97 | } 98 | 99 | res, err := app.run(jobCtx, nil, source.Name, args, false) 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | yamlData := []byte(res.Stdout) 105 | 106 | var ( 107 | format string 108 | key string 109 | ) 110 | 111 | if source.Format != nil { 112 | format = *source.Format 113 | } else { 114 | format = FormatYAML 115 | } 116 | 117 | if source.Key != nil { 118 | key = *source.Key 119 | } 120 | 121 | fragments := []configFragment{ 122 | { 123 | data: yamlData, 124 | key: key, 125 | format: format, 126 | }, 127 | } 128 | 129 | return fragments, nil 130 | } 131 | 132 | func loadFileConfigSource(confCtx *hcl.EvalContext, sourceSpec ConfigSource) ([]configFragment, error) { 133 | var source SourceFile 134 | if err := gohcl2.DecodeBody(sourceSpec.Body, confCtx, &source); err != nil { 135 | return nil, err 136 | } 137 | 138 | format := FormatYAML 139 | 140 | var key string 141 | 142 | if source.Key != nil { 143 | key = *source.Key 144 | } 145 | 146 | var paths []string 147 | 148 | if p := source.Path; p != nil && *p != "" { 149 | paths = append(paths, *p) 150 | } 151 | 152 | paths = append(paths, source.Paths...) 153 | 154 | var fragments []configFragment 155 | 156 | if len(paths) == 0 { 157 | return nil, errors.New("either path or paths must be specified") 158 | } 159 | 160 | for _, path := range paths { 161 | yamlData, err := ioutil.ReadFile(path) 162 | if err != nil { 163 | if source.Default == nil { 164 | return nil, err 165 | } 166 | 167 | yamlData = []byte(*source.Default) 168 | } 169 | 170 | fragments = append(fragments, configFragment{ 171 | data: yamlData, 172 | key: key, 173 | format: format, 174 | }) 175 | } 176 | 177 | return fragments, nil 178 | } 179 | -------------------------------------------------------------------------------- /pkg/app/cty2go.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/hashicorp/hcl/v2" 7 | "github.com/hashicorp/hcl/v2/gohcl" 8 | "github.com/zclconf/go-cty/cty" 9 | "github.com/zclconf/go-cty/cty/gocty" 10 | ) 11 | 12 | func exprMapToGoMap(ctx *hcl.EvalContext, m map[string]hcl.Expression) (map[string]interface{}, error) { 13 | args := map[string]interface{}{} 14 | 15 | for k := range m { 16 | var v cty.Value 17 | if diags := gohcl.DecodeExpression(m[k], ctx, &v); diags.HasErrors() { 18 | return nil, diags 19 | } 20 | 21 | vv, err := ctyToGo(v) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | args[k] = vv 27 | } 28 | 29 | return args, nil 30 | } 31 | 32 | func exprToGoMap(ctx *hcl.EvalContext, expr hcl.Expression) (map[string]interface{}, error) { 33 | args := map[string]interface{}{} 34 | 35 | // We need to explicitly specify that the type of values is DynamicPseudoType. 36 | // 37 | // Otherwise, for e.g. map[string]cty.Value{], DecodeExpression computes the lowest common type for all the values. 38 | // That is, {"foo":true,"bar":"BAR"} would produce cty.Map(cty.String) = map[string]string, 39 | // rather than cty.Map(DynamicPseudoType) = map[string]interface{}. 40 | m := cty.MapValEmpty(cty.DynamicPseudoType) 41 | 42 | if err := gohcl.DecodeExpression(expr, ctx, &m); err != nil { 43 | return nil, err 44 | } 45 | 46 | ctyArgs := m.AsValueMap() 47 | 48 | for k, v := range ctyArgs { 49 | var err error 50 | 51 | args[k], err = ctyToGo(v) 52 | 53 | if err != nil { 54 | return nil, err 55 | } 56 | } 57 | 58 | return args, nil 59 | } 60 | 61 | func ctyToGo(v cty.Value) (interface{}, error) { 62 | var vv interface{} 63 | 64 | switch tpe := v.Type(); tpe { 65 | case cty.String: 66 | var vvv string 67 | 68 | if err := gocty.FromCtyValue(v, &vvv); err != nil { 69 | return nil, err 70 | } 71 | 72 | vv = vvv 73 | case cty.Number: 74 | var vvv int 75 | 76 | if err := gocty.FromCtyValue(v, &vvv); err != nil { 77 | return nil, err 78 | } 79 | 80 | vv = vvv 81 | case cty.Bool: 82 | var vvv bool 83 | 84 | if err := gocty.FromCtyValue(v, &vvv); err != nil { 85 | return nil, err 86 | } 87 | 88 | vv = vvv 89 | case cty.List(cty.String): 90 | var vvv []string 91 | 92 | if err := gocty.FromCtyValue(v, &vvv); err != nil { 93 | return nil, err 94 | } 95 | 96 | vv = vvv 97 | case cty.List(cty.Number): 98 | var vvv []int 99 | 100 | if err := gocty.FromCtyValue(v, &vvv); err != nil { 101 | return nil, err 102 | } 103 | 104 | vv = vvv 105 | case cty.Map(cty.String): 106 | m := map[string]string{} 107 | 108 | if err := gocty.FromCtyValue(v, &v); err != nil { 109 | return nil, err 110 | } 111 | 112 | vv = m 113 | case cty.Map(cty.DynamicPseudoType): 114 | m := map[string]interface{}{} 115 | 116 | for k, v := range v.AsValueMap() { 117 | v, err := ctyToGo(v) 118 | if err != nil { 119 | return nil, err 120 | } 121 | 122 | m[k] = v 123 | } 124 | 125 | vv = m 126 | default: 127 | if tpe.IsTupleType() { 128 | a, err := ctyTupleToGo(v) 129 | if err != nil { 130 | return nil, err 131 | } 132 | 133 | vv = a 134 | } else if tpe.IsObjectType() { 135 | m := map[string]interface{}{} 136 | 137 | for name := range tpe.AttributeTypes() { 138 | attr := v.GetAttr(name) 139 | 140 | v, err := ctyToGo(attr) 141 | if err != nil { 142 | return nil, fmt.Errorf("unable to decoode attribute %q of object: %w", name, err) 143 | } 144 | m[name] = v 145 | } 146 | 147 | vv = m 148 | } else { 149 | return nil, fmt.Errorf("handler for type %s not implemented yet", v.Type().FriendlyName()) 150 | } 151 | } 152 | 153 | return vv, nil 154 | } 155 | 156 | func ctyTupleToGo(tuple cty.Value) (interface{}, error) { 157 | tpe := tuple.Type() 158 | 159 | elemTypes := tpe.TupleElementTypes() 160 | 161 | if len(elemTypes) == 0 { 162 | return []interface{}{}, nil 163 | } 164 | 165 | var lastElemType *cty.Type 166 | 167 | var typeVaries bool 168 | 169 | for i := range elemTypes { 170 | t := &elemTypes[i] 171 | 172 | if lastElemType == nil { 173 | lastElemType = t 174 | } else if !lastElemType.Equals(*t) { 175 | // return nil, fmt.Errorf("handler for tuple with varying element types is not implemented yet: %v", v) 176 | typeVaries = true 177 | 178 | break 179 | } 180 | } 181 | 182 | if typeVaries { 183 | var elems []interface{} 184 | 185 | iter := tuple.ElementIterator() 186 | 187 | for iter.Next() { 188 | _, elemValue := iter.Element() 189 | 190 | elemGo, err := ctyToGo(elemValue) 191 | if err != nil { 192 | return nil, err 193 | } 194 | 195 | elems = append(elems, elemGo) 196 | } 197 | 198 | return elems, nil 199 | } 200 | 201 | var nonEmptyGoSlice interface{} 202 | 203 | switch *lastElemType { 204 | case cty.String: 205 | var strSlice []string 206 | 207 | for i := range elemTypes { 208 | var elem string 209 | 210 | if err := gocty.FromCtyValue(tuple.Index(cty.NumberIntVal(int64(i))), &elem); err != nil { 211 | return nil, err 212 | } 213 | 214 | strSlice = append(strSlice, elem) 215 | } 216 | 217 | nonEmptyGoSlice = strSlice 218 | case cty.Number: 219 | var intSlice []int 220 | 221 | for i := range elemTypes { 222 | var elem int 223 | 224 | if err := gocty.FromCtyValue(tuple.Index(cty.NumberIntVal(int64(i))), &elem); err != nil { 225 | return nil, err 226 | } 227 | 228 | intSlice = append(intSlice, elem) 229 | } 230 | 231 | nonEmptyGoSlice = intSlice 232 | default: 233 | return nil, fmt.Errorf("handler for tuple with element type of %s is not implemented yet: %v", *lastElemType, tuple) 234 | } 235 | 236 | return nonEmptyGoSlice, nil 237 | } 238 | -------------------------------------------------------------------------------- /pkg/app/export.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import "path/filepath" 4 | 5 | func (app *App) moduleName(srcDir string) string { 6 | return "example.com/" + filepath.Base(srcDir) 7 | } 8 | -------------------------------------------------------------------------------- /pkg/app/go_to_cty.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/zclconf/go-cty/cty" 7 | ) 8 | 9 | func goToCty(goV interface{}) (cty.Value, error) { 10 | switch typed := goV.(type) { 11 | case map[string]interface{}: 12 | m := map[string]cty.Value{} 13 | 14 | for k, v := range typed { 15 | var err error 16 | 17 | m[k], err = goToCty(v) 18 | if err != nil { 19 | return cty.DynamicVal, err 20 | } 21 | } 22 | 23 | // cty.MapVal doesn't support empty maps. It panics when encountered an empty map, so... 24 | if len(m) == 0 { 25 | return cty.MapValEmpty(cty.DynamicPseudoType), nil 26 | } 27 | 28 | return cty.MapVal(m), nil 29 | case map[string]string: 30 | return strToStrMapToCty(typed) 31 | case string: 32 | return cty.StringVal(typed), nil 33 | case *string: 34 | if typed == nil { 35 | return cty.NullVal(cty.String), nil 36 | } 37 | 38 | return goToCty(*typed) 39 | case bool: 40 | return cty.BoolVal(typed), nil 41 | case *bool: 42 | if typed == nil { 43 | return cty.NullVal(cty.Bool), nil 44 | } 45 | 46 | return goToCty(*typed) 47 | case int: 48 | return cty.NumberIntVal(int64(typed)), nil 49 | case *int: 50 | if typed == nil { 51 | return cty.NullVal(cty.Number), nil 52 | } 53 | 54 | return goToCty(*typed) 55 | case []string: 56 | var vs []cty.Value 57 | 58 | for _, s := range typed { 59 | vs = append(vs, cty.StringVal(s)) 60 | } 61 | 62 | return cty.ListVal(vs), nil 63 | case *[]string: 64 | if typed == nil { 65 | return cty.ListValEmpty(cty.String), nil 66 | } 67 | 68 | return goToCty(*typed) 69 | case []int: 70 | var vs []cty.Value 71 | 72 | for _, i := range typed { 73 | vs = append(vs, cty.NumberIntVal(int64(i))) 74 | } 75 | 76 | return cty.ListVal(vs), nil 77 | case *[]int: 78 | if typed == nil { 79 | return cty.ListValEmpty(cty.Number), nil 80 | } 81 | 82 | return goToCty(*typed) 83 | case []interface{}: 84 | if len(typed) == 0 { 85 | return cty.ListValEmpty(cty.DynamicPseudoType), nil 86 | } 87 | 88 | var vs []cty.Value 89 | 90 | for _, v := range typed { 91 | vv, err := goToCty(v) 92 | if err != nil { 93 | return cty.DynamicVal, err 94 | } 95 | 96 | vs = append(vs, vv) 97 | } 98 | 99 | return cty.ListVal(vs), nil 100 | default: 101 | return cty.DynamicVal, fmt.Errorf("unsupported type of value %v(%T)", typed, typed) 102 | } 103 | } 104 | 105 | func strToStrMapToCty(typed map[string]string) (cty.Value, error) { 106 | m := map[string]cty.Value{} 107 | 108 | for k, v := range typed { 109 | var err error 110 | 111 | m[k], err = goToCty(v) 112 | if err != nil { 113 | return cty.DynamicVal, err 114 | } 115 | } 116 | 117 | // cty.MapVal doesn't support empty maps. It panics when encountered an empty map, so... 118 | if len(m) == 0 { 119 | return cty.MapValEmpty(cty.String), nil 120 | } 121 | 122 | return cty.MapVal(m), nil 123 | } 124 | -------------------------------------------------------------------------------- /pkg/app/job.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | gohcl2 "github.com/hashicorp/hcl/v2/gohcl" 5 | ) 6 | 7 | type eitherJobRun struct { 8 | static *StaticRun 9 | dynamic *DynamicRun 10 | } 11 | 12 | type jobRun struct { 13 | Name string 14 | Args map[string]interface{} 15 | Skipped bool 16 | } 17 | 18 | func staticRunToJob(jobCtx *JobContext, run *StaticRun) (*jobRun, error) { 19 | localArgs, err := exprMapToGoMap(jobCtx.evalContext, run.Args) 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | args := map[string]interface{}{} 25 | 26 | for k, v := range jobCtx.globalArgs { 27 | args[k] = v 28 | } 29 | 30 | for k, v := range localArgs { 31 | args[k] = v 32 | } 33 | 34 | return &jobRun{ 35 | Name: run.Name, 36 | Args: args, 37 | }, nil 38 | } 39 | 40 | func dynamicRunToJob(jobCtx *JobContext, run *DynamicRun) (*jobRun, error) { 41 | localArgs, err := exprToGoMap(jobCtx.evalContext, run.Args) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | if !IsExpressionEmpty(run.Condition) { 47 | var condition bool 48 | 49 | if diags := gohcl2.DecodeExpression(run.Condition, jobCtx.evalContext, &condition); diags.HasErrors() { 50 | return nil, diags 51 | } 52 | 53 | if !condition { 54 | return &jobRun{ 55 | Name: run.Job, 56 | Skipped: true, 57 | }, nil 58 | } 59 | } 60 | 61 | args := map[string]interface{}{} 62 | 63 | for k, v := range jobCtx.globalArgs { 64 | args[k] = v 65 | } 66 | 67 | for k, v := range localArgs { 68 | args[k] = v 69 | } 70 | 71 | return &jobRun{ 72 | Name: run.Job, 73 | Args: args, 74 | }, nil 75 | } 76 | -------------------------------------------------------------------------------- /pkg/app/load_test.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | ) 8 | 9 | func TestNewImportFunc(t *testing.T) { 10 | type testcase struct { 11 | subject string 12 | path string 13 | want string 14 | } 15 | 16 | testcases := []testcase{ 17 | { 18 | subject: "relative path", 19 | path: "foo/bar", 20 | want: "sub/foo/bar", 21 | }, 22 | { 23 | subject: "git url", 24 | path: "git::ssh://git@github.com/mumoshu/variant2@examples/advanced/import/foo?ref=master", 25 | want: "git::ssh://git@github.com/mumoshu/variant2@examples/advanced/import/foo?ref=master", 26 | }, 27 | { 28 | subject: "absolute path", 29 | path: "/a/b/c", 30 | want: "/a/b/c", 31 | }, 32 | } 33 | 34 | for i := range testcases { 35 | tc := testcases[i] 36 | 37 | t.Run(tc.subject, func(t *testing.T) { 38 | a := &App{} 39 | 40 | f := NewImportFunc("sub", func(path string) (*App, error) { 41 | if d := cmp.Diff(tc.want, path); d != "" { 42 | t.Errorf("%s: %s", tc.subject, d) 43 | } 44 | 45 | return a, nil 46 | }) 47 | 48 | r, _ := f(tc.path) 49 | 50 | if r != a { 51 | t.Fatalf("%s: unexpected result returned", tc.subject) 52 | } 53 | }) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pkg/app/log.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "strings" 7 | "sync" 8 | "time" 9 | 10 | "github.com/zclconf/go-cty/cty" 11 | "golang.org/x/xerrors" 12 | ) 13 | 14 | type Event struct { 15 | Type string 16 | Time time.Time 17 | Run *RunEvent 18 | Exec *ExecEvent 19 | } 20 | 21 | type RunEvent struct { 22 | Job string 23 | Args map[string]interface{} 24 | } 25 | 26 | type ExecEvent struct { 27 | Command string 28 | Args []string 29 | } 30 | 31 | func (evt Event) toCty() cty.Value { 32 | m := map[string]cty.Value{ 33 | "type": cty.StringVal(evt.Type), 34 | } 35 | m["time"] = cty.StringVal(evt.Time.Format(time.RFC3339)) 36 | 37 | if evt.Run != nil { 38 | m["run"] = evt.Run.toCty() 39 | } 40 | 41 | if evt.Exec != nil { 42 | m["exec"] = evt.Exec.toCty() 43 | } 44 | 45 | return cty.ObjectVal(m) 46 | } 47 | 48 | func (e *RunEvent) toCty() cty.Value { 49 | var args cty.Value 50 | 51 | if len(e.Args) > 0 { 52 | var err error 53 | 54 | args, err = goToCty(e.Args) 55 | if err != nil { 56 | panic(err) 57 | } 58 | } else { 59 | // We can't call goToCty for empty map because it results in go-cty panic with "must not call MapVal with empty map" 60 | args = cty.MapValEmpty(cty.DynamicPseudoType) 61 | } 62 | 63 | return cty.ObjectVal(map[string]cty.Value{ 64 | "job": cty.StringVal(e.Job), 65 | "args": args, 66 | }) 67 | } 68 | 69 | func (e *ExecEvent) toCty() cty.Value { 70 | vals := []cty.Value{} 71 | for _, a := range e.Args { 72 | vals = append(vals, cty.StringVal(a)) 73 | } 74 | 75 | return cty.ObjectVal(map[string]cty.Value{ 76 | "command": cty.StringVal(e.Command), 77 | "args": cty.ListVal(vals), 78 | }) 79 | } 80 | 81 | type EventLogger struct { 82 | lastIndex int 83 | 84 | Command string 85 | Args, Opts map[string]interface{} 86 | 87 | Stream string 88 | 89 | Stderr io.Writer 90 | 91 | Events []Event 92 | 93 | collectors map[int]*LogCollector 94 | 95 | collectorsMutex sync.Mutex 96 | eventsMutex sync.Mutex 97 | } 98 | 99 | func NewEventLogger(cmd string, args map[string]interface{}, opts map[string]interface{}) *EventLogger { 100 | return &EventLogger{ 101 | collectors: map[int]*LogCollector{}, 102 | Events: []Event{}, 103 | Command: cmd, 104 | Args: args, 105 | Opts: opts, 106 | } 107 | } 108 | 109 | func (l *EventLogger) LogRun(job string, args map[string]interface{}) error { 110 | return l.append(Event{Type: "run", Time: time.Now(), Run: &RunEvent{ 111 | Job: job, 112 | Args: args, 113 | }}) 114 | } 115 | 116 | func (l *EventLogger) LogExec(cmd string, args []string) error { 117 | return l.append(Event{Type: "exec", Time: time.Now(), Exec: &ExecEvent{ 118 | Command: cmd, 119 | Args: args, 120 | }}) 121 | } 122 | 123 | func (l *EventLogger) append(evt Event) error { 124 | l.eventsMutex.Lock() 125 | l.Events = append(l.Events, evt) 126 | l.eventsMutex.Unlock() 127 | 128 | l.collectorsMutex.Lock() 129 | defer l.collectorsMutex.Unlock() 130 | 131 | for _, c := range l.collectors { 132 | line, err := c.Collect(evt) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | // Non-nil line means that any collect block's condition matched the logged event 138 | if line != nil && l.Stream == "stderr" { 139 | if _, err := l.Stderr.Write([]byte(*line + "\n")); err != nil { 140 | return xerrors.Errorf("wrirting stderr: %w", err) 141 | } 142 | } 143 | } 144 | 145 | return nil 146 | } 147 | 148 | func (l *EventLogger) Register(logCollector LogCollector) func() error { 149 | id := l.lastIndex + 1 150 | l.lastIndex = id 151 | 152 | l.collectorsMutex.Lock() 153 | l.collectors[id] = &logCollector 154 | l.collectorsMutex.Unlock() 155 | 156 | return func() error { 157 | defer func() { 158 | l.collectorsMutex.Lock() 159 | delete(l.collectors, id) 160 | l.collectorsMutex.Unlock() 161 | }() 162 | 163 | var file string 164 | 165 | if logCollector.FilePath == "" { 166 | tmpFile, _ := ioutil.TempFile("", "tmp") 167 | file = tmpFile.Name() 168 | } else { 169 | file = logCollector.FilePath 170 | } 171 | 172 | //nolint:gosec 173 | if err := ioutil.WriteFile(file, []byte(strings.Join(logCollector.lines, "\n")), 0o644); err != nil { 174 | return xerrors.Errorf("writing %s: %w", file, err) 175 | } 176 | 177 | log := Log{ 178 | File: file, 179 | } 180 | 181 | return logCollector.ForwardFn(log) 182 | } 183 | } 184 | 185 | type Log struct { 186 | File string 187 | } 188 | 189 | type LogCollector struct { 190 | FilePath string 191 | CollectFn func(Event) (*string, bool, error) 192 | ForwardFn func(log Log) error 193 | lines []string 194 | } 195 | 196 | func (c *LogCollector) Collect(evt Event) (*string, error) { 197 | text, shouldCollect, err := c.CollectFn(evt) 198 | if err != nil { 199 | return nil, err 200 | } 201 | 202 | if shouldCollect { 203 | if c.lines == nil { 204 | c.lines = []string{} 205 | } 206 | 207 | c.lines = append(c.lines, *text) 208 | } 209 | 210 | return text, nil 211 | } 212 | -------------------------------------------------------------------------------- /pkg/app/run.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | 7 | "github.com/hashicorp/hcl/v2" 8 | "github.com/hashicorp/hcl/v2/gohcl" 9 | ) 10 | 11 | func (app *App) runJobInBody(l *EventLogger, jobCtx *JobContext, body hcl.Body, streamOutput bool) (*Result, bool, error) { 12 | var runs []eitherJobRun 13 | 14 | var lazyStaticRun LazyStaticRun 15 | 16 | sErr := gohcl.DecodeBody(body, jobCtx.evalContext, &lazyStaticRun) 17 | 18 | //nolint:nestif 19 | if sErr.HasErrors() { 20 | var lazyDynamicRun LazyDynamicRun 21 | 22 | dErr := gohcl.DecodeBody(body, jobCtx.evalContext, &lazyDynamicRun) 23 | 24 | if dErr != nil { 25 | sErrMsg := sErr.Error() 26 | if !strings.Contains(sErrMsg, "Missing run block") && !strings.Contains(sErrMsg, "Missing name for run") { 27 | return nil, false, sErr 28 | } 29 | 30 | dErrMsg := dErr.Error() 31 | if !strings.Contains(dErrMsg, "Missing run block") { 32 | return nil, false, dErr 33 | } 34 | } else { 35 | for i := range lazyDynamicRun.Run { 36 | r := lazyDynamicRun.Run[i] 37 | 38 | either := eitherJobRun{} 39 | 40 | either.dynamic = &r 41 | 42 | runs = append(runs, either) 43 | } 44 | } 45 | } else { 46 | for i := range lazyStaticRun.Run { 47 | r := lazyStaticRun.Run[i] 48 | 49 | either := eitherJobRun{} 50 | 51 | either.static = &r 52 | 53 | runs = append(runs, either) 54 | } 55 | } 56 | 57 | if len(runs) == 0 { 58 | return nil, false, nil 59 | } 60 | 61 | var results []*Result 62 | 63 | for _, r := range runs { 64 | res, err := app.runJobAndUpdateContext(l, jobCtx, r, new(sync.Mutex), streamOutput) 65 | if err != nil { 66 | return res, true, err 67 | } 68 | 69 | if res == nil { 70 | return res, true, nil 71 | } 72 | 73 | if !res.Skipped { 74 | results = append(results, res) 75 | } 76 | } 77 | 78 | if len(results) == 0 { 79 | return nil, true, nil 80 | } 81 | 82 | return aggregateResults(results), true, nil 83 | } 84 | 85 | func aggregateResults(results []*Result) *Result { 86 | aggregated := *results[len(results)-1] 87 | 88 | aggregated.Stdout = "" 89 | aggregated.Stderr = "" 90 | 91 | for _, r := range results { 92 | if r.Stdout != "" { 93 | aggregated.Stdout += r.Stdout 94 | } 95 | 96 | if r.Stderr != "" { 97 | aggregated.Stderr += r.Stderr 98 | } 99 | } 100 | 101 | return &aggregated 102 | } 103 | -------------------------------------------------------------------------------- /pkg/app/run_args.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import "github.com/hashicorp/hcl/v2" 4 | 5 | func buildArgsFromExpr(jobCtx *JobContext, expr hcl.Expression) (map[string]interface{}, error) { 6 | localArgs, err := exprToGoMap(jobCtx.evalContext, expr) 7 | if err != nil { 8 | return nil, err 9 | } 10 | 11 | args := map[string]interface{}{} 12 | 13 | for k, v := range jobCtx.globalArgs { 14 | args[k] = v 15 | } 16 | 17 | for k, v := range localArgs { 18 | args[k] = v 19 | } 20 | 21 | return args, nil 22 | } 23 | -------------------------------------------------------------------------------- /pkg/app/source.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/zclconf/go-cty/cty" 8 | "golang.org/x/sync/errgroup" 9 | "golang.org/x/xerrors" 10 | 11 | "github.com/mumoshu/variant2/pkg/kube" 12 | "github.com/mumoshu/variant2/pkg/source" 13 | ) 14 | 15 | func (app *App) checkoutSources(_ *EventLogger, jobCtx *JobContext, sources []Source, concurrency int) error { 16 | if len(sources) == 0 { 17 | return nil 18 | } 19 | 20 | if app.sourceClient == nil { 21 | err := func() error { 22 | app.initMu.Lock() 23 | defer app.initMu.Unlock() 24 | 25 | client, err := kube.NewClient() 26 | if err != nil { 27 | return err 28 | } 29 | 30 | app.sourceClient = &source.Client{Client: client} 31 | 32 | return nil 33 | }() 34 | if err != nil { 35 | return err 36 | } 37 | } 38 | 39 | type result struct { 40 | id string 41 | dir string 42 | } 43 | 44 | sourceCh := make(chan Source) 45 | 46 | resultCh := make(chan result) 47 | 48 | sourceWorkers, ctx := errgroup.WithContext(context.Background()) 49 | 50 | for i := 0; i < concurrency; i++ { 51 | sourceWorkers.Go(func() error { 52 | for src := range sourceCh { 53 | dir, err := app.sourceClient.ExtractSource(ctx, src.Kind, src.Namepsace, src.Name) 54 | if err != nil { 55 | return xerrors.Errorf("extracting source: %w", err) 56 | } 57 | 58 | resultCh <- result{id: src.ID, dir: dir} 59 | } 60 | 61 | return nil 62 | }) 63 | } 64 | 65 | results := map[string]cty.Value{} 66 | 67 | wg := &sync.WaitGroup{} 68 | wg.Add(1) 69 | 70 | go func() { 71 | for src := range resultCh { 72 | results[src.id] = cty.MapVal(map[string]cty.Value{ 73 | "dir": cty.StringVal(src.dir), 74 | }) 75 | } 76 | 77 | wg.Done() 78 | }() 79 | 80 | for _, src := range sources { 81 | sourceCh <- src 82 | } 83 | 84 | close(sourceCh) 85 | 86 | if err := sourceWorkers.Wait(); err != nil { 87 | return xerrors.Errorf("waiting for workers: %w", err) 88 | } 89 | 90 | close(resultCh) 91 | 92 | wg.Wait() 93 | 94 | if len(results) > 0 { 95 | // Cuz calling cty.MapVal on an empty map panics by its nature 96 | jobCtx.evalContext.Variables["source"] = cty.MapVal(results) 97 | } 98 | 99 | return nil 100 | } 101 | -------------------------------------------------------------------------------- /pkg/app/survey.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/AlecAivazis/survey/v2" 9 | "github.com/zclconf/go-cty/cty" 10 | ) 11 | 12 | type PendingInput struct { 13 | Name string 14 | Description *string 15 | Type cty.Type 16 | } 17 | 18 | func MakeQuestions(pendingOptions []PendingInput) ([]*survey.Question, map[string]survey.Transformer, error) { 19 | qs := []*survey.Question{} 20 | 21 | transformers := map[string]survey.Transformer{} 22 | 23 | for _, op := range pendingOptions { 24 | name := op.Name 25 | 26 | var msg string 27 | 28 | var description string 29 | 30 | if op.Description != nil { 31 | description = *op.Description 32 | } 33 | 34 | msg = name 35 | 36 | var validate survey.Validator 37 | 38 | var transform survey.Transformer 39 | 40 | var prompt survey.Prompt 41 | 42 | switch op.Type { 43 | case cty.String: 44 | prompt = &survey.Input{ 45 | Message: msg, 46 | Help: description, 47 | } 48 | case cty.Number: 49 | prompt = &survey.Input{ 50 | Message: msg, 51 | Help: description, 52 | } 53 | 54 | transform = func(ans interface{}) (newAns interface{}) { 55 | i, _ := strconv.Atoi(ans.(string)) 56 | 57 | return i 58 | } 59 | 60 | validate = func(ans interface{}) error { 61 | switch v := ans.(type) { 62 | case string: 63 | if _, err := strconv.Atoi(v); err != nil { 64 | return fmt.Errorf("option %q: %w", name, err) 65 | } 66 | default: 67 | return fmt.Errorf("option %q: number: unexpected type of input %T", name, v) 68 | } 69 | 70 | return nil 71 | } 72 | case cty.Bool: 73 | prompt = &survey.Confirm{ 74 | Message: msg, 75 | Help: description, 76 | Default: false, 77 | } 78 | case cty.List(cty.String): 79 | prompt = &survey.Multiline{ 80 | Message: msg, 81 | Help: description, 82 | } 83 | 84 | transform = func(ans interface{}) (newAns interface{}) { 85 | lines := strings.Split(ans.(string), "\n") 86 | 87 | return lines 88 | } 89 | 90 | validate = func(ans interface{}) error { 91 | switch v := ans.(type) { 92 | case string: 93 | default: 94 | return fmt.Errorf("option %q: list(string): unexpected type of input %T", name, v) 95 | } 96 | 97 | return nil 98 | } 99 | case cty.List(cty.Number): 100 | prompt = &survey.Multiline{ 101 | Message: msg, 102 | Help: description, 103 | } 104 | 105 | transform = func(ans interface{}) (newAns interface{}) { 106 | lines := strings.Split(ans.(string), "\n") 107 | 108 | var ints []int 109 | 110 | for _, line := range lines { 111 | i, _ := strconv.Atoi(line) 112 | ints = append(ints, i) 113 | } 114 | 115 | return ints 116 | } 117 | 118 | validate = func(ans interface{}) error { 119 | switch v := ans.(type) { 120 | case string: 121 | vs := strings.Split(v, "\n") 122 | 123 | for _, a := range vs { 124 | _, err := strconv.Atoi(a) 125 | if err != nil { 126 | return fmt.Errorf("option %q: list(number): atoi: %w", name, err) 127 | } 128 | } 129 | default: 130 | return fmt.Errorf("option %q: list(number): unexpected type of input %T", name, v) 131 | } 132 | 133 | return nil 134 | } 135 | default: 136 | return nil, nil, fmt.Errorf("option %q: unexpected type %q", op.Name, op.Type.FriendlyName()) 137 | } 138 | 139 | validators := []survey.Validator{survey.Required} 140 | 141 | if validate != nil { 142 | validators = append(validators, validate) 143 | } 144 | 145 | qs = append(qs, &survey.Question{ 146 | Name: name, 147 | Prompt: prompt, 148 | Validate: survey.ComposeValidators(validators...), 149 | }) 150 | 151 | if transform != nil { 152 | transformers[name] = transform 153 | } 154 | } 155 | 156 | return qs, transformers, nil 157 | } 158 | 159 | type SetOptsFunc func(opts map[string]cty.Value, pendingOptions []PendingInput) error 160 | 161 | func DefaultSetOpts(opts map[string]cty.Value, pendingOptions []PendingInput) error { 162 | qs, transformers, err := MakeQuestions(pendingOptions) 163 | if err != nil { 164 | return err 165 | } 166 | 167 | res := make(map[string]interface{}) 168 | 169 | if err := survey.Ask(qs, &res); err != nil { 170 | return err 171 | } 172 | 173 | return SetOptsFromMap(transformers, opts, res) 174 | } 175 | 176 | func SetOptsFromMap(transformers map[string]survey.Transformer, opts map[string]cty.Value, res map[string]interface{}) error { 177 | for k, v := range res { 178 | t, ok := transformers[k] 179 | 180 | var ans interface{} 181 | 182 | if ok { 183 | ans = t(v) 184 | } else { 185 | ans = v 186 | } 187 | 188 | switch v := ans.(type) { 189 | case int: 190 | opts[k] = cty.NumberIntVal(int64(v)) 191 | case string: 192 | opts[k] = cty.StringVal(v) 193 | case []string: 194 | vs := []cty.Value{} 195 | for _, s := range v { 196 | vs = append(vs, cty.StringVal(s)) 197 | } 198 | 199 | opts[k] = cty.ListVal(vs) 200 | case []int: 201 | vs := []cty.Value{} 202 | for _, i := range v { 203 | vs = append(vs, cty.NumberIntVal(int64(i))) 204 | } 205 | 206 | opts[k] = cty.ListVal(vs) 207 | case bool: 208 | opts[k] = cty.BoolVal(v) 209 | default: 210 | return fmt.Errorf("option %q: parsing answer: unexpected type %T", k, v) 211 | } 212 | } 213 | 214 | return nil 215 | } 216 | -------------------------------------------------------------------------------- /pkg/app/testdeps_deps.go: -------------------------------------------------------------------------------- 1 | // Copied from go/1.13.4/libexec/src/testing/internal/testdeps/deps.go 2 | 3 | // Copyright 2016 The Go Authors. All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | // Package testdeps provides access to dependencies needed by test execution. 8 | // 9 | // This package is imported by the generated main package, which passes 10 | // TestDeps into testing.Main. This allows tests to use packages at run time 11 | // without making those packages direct dependencies of package testing. 12 | // Direct dependencies of package testing are harder to write tests for. 13 | 14 | //nolint 15 | package app 16 | 17 | import ( 18 | "bufio" 19 | "io" 20 | "regexp" 21 | "runtime/pprof" 22 | "strings" 23 | "sync" 24 | ) 25 | 26 | // TestDeps is an implementation of the testing.testDeps interface, 27 | // suitable for passing to testing.MainStart. 28 | type TestDeps struct{} 29 | 30 | var ( 31 | matchPat string 32 | matchRe *regexp.Regexp 33 | ) 34 | 35 | func (TestDeps) MatchString(pat, str string) (result bool, err error) { 36 | if matchRe == nil || matchPat != pat { 37 | matchPat = pat 38 | matchRe, err = regexp.Compile(matchPat) 39 | if err != nil { 40 | return 41 | } 42 | } 43 | return matchRe.MatchString(str), nil 44 | } 45 | 46 | func (TestDeps) StartCPUProfile(w io.Writer) error { 47 | return pprof.StartCPUProfile(w) 48 | } 49 | 50 | func (TestDeps) StopCPUProfile() { 51 | pprof.StopCPUProfile() 52 | } 53 | 54 | func (TestDeps) WriteProfileTo(name string, w io.Writer, debug int) error { 55 | return pprof.Lookup(name).WriteTo(w, debug) 56 | } 57 | 58 | func (TestDeps) SetPanicOnExit0(bool) {} 59 | 60 | // ImportPath is the import path of the testing binary, set by the generated main function. 61 | var ImportPath string 62 | 63 | func (TestDeps) ImportPath() string { 64 | return ImportPath 65 | } 66 | 67 | // testLog implements testlog.Interface, logging actions by package os. 68 | type testLog struct { 69 | mu sync.Mutex 70 | w *bufio.Writer 71 | set bool 72 | } 73 | 74 | func (l *testLog) Getenv(key string) { 75 | l.add("getenv", key) 76 | } 77 | 78 | func (l *testLog) Open(name string) { 79 | l.add("open", name) 80 | } 81 | 82 | func (l *testLog) Stat(name string) { 83 | l.add("stat", name) 84 | } 85 | 86 | func (l *testLog) Chdir(name string) { 87 | l.add("chdir", name) 88 | } 89 | 90 | // add adds the (op, name) pair to the test log. 91 | func (l *testLog) add(op, name string) { 92 | if strings.Contains(name, "\n") || name == "" { 93 | return 94 | } 95 | 96 | l.mu.Lock() 97 | defer l.mu.Unlock() 98 | if l.w == nil { 99 | return 100 | } 101 | l.w.WriteString(op) 102 | l.w.WriteByte(' ') 103 | l.w.WriteString(name) 104 | l.w.WriteByte('\n') 105 | } 106 | 107 | var log testLog 108 | 109 | func (TestDeps) StartTestLog(w io.Writer) { 110 | log.mu.Lock() 111 | log.w = bufio.NewWriter(w) 112 | if !log.set { 113 | // Tests that define TestMain and then run m.Run multiple times 114 | // will call StartTestLog/StopTestLog multiple times. 115 | // Checking log.set avoids calling testlog.SetLogger multiple times 116 | // (which will panic) and also avoids writing the header multiple times. 117 | log.set = true 118 | SetLogger(&log) 119 | log.w.WriteString("# test log\n") // known to cmd/go/internal/test/test.go 120 | } 121 | log.mu.Unlock() 122 | } 123 | 124 | func (TestDeps) StopTestLog() error { 125 | log.mu.Lock() 126 | defer log.mu.Unlock() 127 | err := log.w.Flush() 128 | log.w = nil 129 | return err 130 | } 131 | -------------------------------------------------------------------------------- /pkg/app/testing_match.go: -------------------------------------------------------------------------------- 1 | // Copied from go/1.13.4/libexec/src/testing/match.go 2 | 3 | //nolint 4 | package app 5 | 6 | import "strconv" 7 | 8 | // rewrite rewrites a subname to having only printable characters and no white 9 | // space. 10 | func rewrite(s string) string { 11 | b := []byte{} 12 | for _, r := range s { 13 | switch { 14 | case isSpace(r): 15 | b = append(b, '_') 16 | case !strconv.IsPrint(r): 17 | s := strconv.QuoteRune(r) 18 | b = append(b, s[1:len(s)-1]...) 19 | default: 20 | b = append(b, string(r)...) 21 | } 22 | } 23 | return string(b) 24 | } 25 | 26 | func isSpace(r rune) bool { 27 | if r < 0x2000 { 28 | switch r { 29 | // Note: not the same as Unicode Z class. 30 | case '\t', '\n', '\v', '\f', '\r', ' ', 0x85, 0xA0, 0x1680: 31 | return true 32 | } 33 | } else { 34 | if r <= 0x200a { 35 | return true 36 | } 37 | switch r { 38 | case 0x2028, 0x2029, 0x202f, 0x205f, 0x3000: 39 | return true 40 | } 41 | } 42 | return false 43 | } 44 | -------------------------------------------------------------------------------- /pkg/app/testlog_log.go: -------------------------------------------------------------------------------- 1 | // Copied from go/1.13.4/libexec/src/internal/testlog/log.go 2 | // 3 | // Copyright 2017 The Go Authors. All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | // Package testlog provides a back-channel communication path 8 | // between tests and package os, so that cmd/go can see which 9 | // environment variables and files a test consults. 10 | 11 | //nolint 12 | package app 13 | 14 | import "sync/atomic" 15 | 16 | // Interface is the interface required of test loggers. 17 | // The os package will invoke the interface's methods to indicate that 18 | // it is inspecting the given environment variables or files. 19 | // Multiple goroutines may call these methods simultaneously. 20 | type Interface interface { 21 | Getenv(key string) 22 | Stat(file string) 23 | Open(file string) 24 | Chdir(dir string) 25 | } 26 | 27 | // logger is the current logger Interface. 28 | // We use an atomic.Value in case test startup 29 | // is racing with goroutines started during init. 30 | // That must not cause a race detector failure, 31 | // although it will still result in limited visibility 32 | // into exactly what those goroutines do. 33 | var logger atomic.Value 34 | 35 | // SetLogger sets the test logger implementation for the current process. 36 | // It must be called only once, at process startup. 37 | func SetLogger(impl Interface) { 38 | if logger.Load() != nil { 39 | panic("testlog: SetLogger must be called only once") 40 | } 41 | logger.Store(&impl) 42 | } 43 | 44 | // Logger returns the current test logger implementation. 45 | // It returns nil if there is no logger. 46 | func Logger() Interface { 47 | impl := logger.Load() 48 | if impl == nil { 49 | return nil 50 | } 51 | return *impl.(*Interface) 52 | } 53 | 54 | // Getenv calls Logger().Getenv, if a logger has been set. 55 | func Getenv(name string) { 56 | if log := Logger(); log != nil { 57 | log.Getenv(name) 58 | } 59 | } 60 | 61 | // Open calls Logger().Open, if a logger has been set. 62 | func Open(name string) { 63 | if log := Logger(); log != nil { 64 | log.Open(name) 65 | } 66 | } 67 | 68 | // Stat calls Logger().Stat, if a logger has been set. 69 | func Stat(name string) { 70 | if log := Logger(); log != nil { 71 | log.Stat(name) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /pkg/app/trace.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "encoding/json" 5 | ) 6 | 7 | func (app *App) newTracingLogCollector() LogCollector { 8 | logCollector := LogCollector{ 9 | CollectFn: func(evt Event) (*string, bool, error) { 10 | bs, err := json.Marshal(evt) 11 | if err != nil { 12 | return nil, false, err 13 | } 14 | 15 | if _, err := app.Stderr.Write(append([]byte("TRACE\t"), bs...)); err != nil { 16 | panic(err) 17 | } 18 | 19 | return nil, false, nil 20 | }, 21 | ForwardFn: func(log Log) error { 22 | return nil 23 | }, 24 | } 25 | 26 | return logCollector 27 | } 28 | -------------------------------------------------------------------------------- /pkg/app/types.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | 7 | "github.com/hashicorp/hcl/v2" 8 | "github.com/zclconf/go-cty/cty/function" 9 | 10 | "github.com/mumoshu/variant2/pkg/source" 11 | ) 12 | 13 | type Config struct { 14 | Name string `hcl:"name,label"` 15 | 16 | Sources []ConfigSource `hcl:"source,block"` 17 | } 18 | 19 | type ConfigSource struct { 20 | Type string `hcl:"type,label"` 21 | 22 | Body hcl.Body `hcl:",remain"` 23 | } 24 | 25 | type SourceFile struct { 26 | Path *string `hcl:"path,attr"` 27 | Paths []string `hcl:"paths,optional"` 28 | Default *string `hcl:"default,attr"` 29 | Key *string `hcl:"key,attr"` 30 | } 31 | 32 | type Step struct { 33 | Name string `hcl:"name,label"` 34 | 35 | Run StaticRun `hcl:"run,block"` 36 | 37 | Needs *[]string `hcl:"need,attr"` 38 | } 39 | 40 | type Exec struct { 41 | Command hcl.Expression `hcl:"command,attr"` 42 | 43 | Args hcl.Expression `hcl:"args,attr"` 44 | Env hcl.Expression `hcl:"env,attr"` 45 | Dir hcl.Expression `hcl:"dir,attr"` 46 | 47 | Interactive *bool `hcl:"interactive,attr"` 48 | } 49 | 50 | type DependsOn struct { 51 | Name string `hcl:"name,label"` 52 | 53 | Items hcl.Expression `hcl:"items,attr"` 54 | Args hcl.Expression `hcl:"args,attr"` 55 | } 56 | 57 | type LazyStaticRun struct { 58 | Run []StaticRun `hcl:"run,block"` 59 | } 60 | 61 | type StaticRun struct { 62 | Name string `hcl:"name,label"` 63 | 64 | Args map[string]hcl.Expression `hcl:",remain"` 65 | } 66 | 67 | type LazyDynamicRun struct { 68 | Run []DynamicRun `hcl:"run,block"` 69 | } 70 | 71 | type DynamicRun struct { 72 | Job string `hcl:"job,attr"` 73 | Args hcl.Expression `hcl:"with,attr"` 74 | Condition hcl.Expression `hcl:"condition,attr"` 75 | } 76 | 77 | type Parameter struct { 78 | Name string `hcl:"name,label"` 79 | 80 | Type hcl.Expression `hcl:"type,attr"` 81 | Default hcl.Expression `hcl:"default,attr"` 82 | Envs []EnvSource `hcl:"env,block"` 83 | 84 | Description *string `hcl:"description,attr"` 85 | } 86 | 87 | type EnvSource struct { 88 | Name string `hcl:"name,label"` 89 | } 90 | 91 | type SourceJob struct { 92 | Name string `hcl:"name,attr"` 93 | // This results in "no cty.Type for hcl.Expression" error 94 | // Arguments map[string]hcl2.Expression `hcl:"args,attr"` 95 | Args hcl.Expression `hcl:"args,attr"` 96 | Format *string `hcl:"format,attr"` 97 | Key *string `hcl:"key,attr"` 98 | } 99 | 100 | type OptionSpec struct { 101 | Name string `hcl:"name,label"` 102 | 103 | Type hcl.Expression `hcl:"type,attr"` 104 | Default hcl.Expression `hcl:"default,attr"` 105 | Description *string `hcl:"description,attr"` 106 | Short *string `hcl:"short,attr"` 107 | } 108 | 109 | type Variable struct { 110 | Name string `hcl:"name,label"` 111 | 112 | Type hcl.Expression `hcl:"type,attr"` 113 | Value hcl.Expression `hcl:"value,attr"` 114 | } 115 | 116 | type JobSpec struct { 117 | // Type string `hcl:"type,label"` 118 | Name string `hcl:"name,label"` 119 | 120 | Version *string `hcl:"version,attr"` 121 | 122 | Module hcl.Expression `hcl:"module,attr"` 123 | 124 | Description *string `hcl:"description,attr"` 125 | Parameters []Parameter `hcl:"parameter,block"` 126 | Options []OptionSpec `hcl:"option,block"` 127 | Configs []Config `hcl:"config,block"` 128 | Secrets []Config `hcl:"secret,block"` 129 | Variables []Variable `hcl:"variable,block"` 130 | 131 | Concurrency hcl.Expression `hcl:"concurrency,attr"` 132 | 133 | SourceLocator hcl.Expression `hcl:"__source_locator,attr"` 134 | 135 | Deps []DependsOn `hcl:"depends_on,block"` 136 | Exec *Exec `hcl:"exec,block"` 137 | Assert []Assert `hcl:"assert,block"` 138 | Fail hcl.Expression `hcl:"fail,attr"` 139 | Import *string `hcl:"import,attr"` 140 | Imports *[]string `hcl:"imports,attr"` 141 | 142 | // Private hides the job from `variant run -h` when set to true 143 | Private *bool `hcl:"private,attr"` 144 | 145 | Log *LogSpec `hcl:"log,block"` 146 | 147 | Steps []Step `hcl:"step,block"` 148 | 149 | Body hcl.Body `hcl:",remain"` 150 | 151 | Sources []Source `hcl:"source,block"` 152 | } 153 | 154 | type Source struct { 155 | ID string `hcl:"id,label"` 156 | // Kind is the kind of source-controller source kind like "GitRepository" and "Bucket" 157 | Kind string `hcl:"kind,attr"` 158 | // Namespace is the K8s namespace of the source-controller source 159 | Namepsace string `hcl:"namespace,attr"` 160 | // Name defaults to ID 161 | Name string `hcl:"name,attr"` 162 | } 163 | 164 | type LogSpec struct { 165 | File hcl.Expression `hcl:"file,attr"` 166 | Stream hcl.Expression `hcl:"stream,attr"` 167 | Collects []Collect `hcl:"collect,block"` 168 | Forwards []Forward `hcl:"forward,block"` 169 | } 170 | 171 | type Collect struct { 172 | Condition hcl.Expression `hcl:"condition,attr"` 173 | Format hcl.Expression `hcl:"format,attr"` 174 | } 175 | 176 | type Forward struct { 177 | Run *StaticRun `hcl:"run,block"` 178 | } 179 | 180 | type Assert struct { 181 | Name string `hcl:"name,label"` 182 | 183 | Condition hcl.Expression `hcl:"condition,attr"` 184 | } 185 | 186 | type HCL2Config struct { 187 | Jobs []JobSpec `hcl:"job,block"` 188 | Tests []Test `hcl:"test,block"` 189 | JobSpec `hcl:",remain"` 190 | } 191 | 192 | type Test struct { 193 | Name string `hcl:"name,label"` 194 | 195 | Variables []Variable `hcl:"variable,block"` 196 | Cases []Case `hcl:"case,block"` 197 | Run StaticRun `hcl:"run,block"` 198 | Assert []Assert `hcl:"assert,block"` 199 | 200 | SourceLocator hcl.Expression `hcl:"__source_locator,attr"` 201 | 202 | ExpectedExecs []Expect `hcl:"expect,block"` 203 | } 204 | 205 | type Expect struct { 206 | // Type must be `exec` for now 207 | Type string `hcl:"type,label"` 208 | 209 | Command hcl.Expression `hcl:"command,attr"` 210 | Args hcl.Expression `hcl:"args,attr"` 211 | Dir hcl.Expression `hcl:"dir,attr"` 212 | } 213 | 214 | type expectedExec struct { 215 | Command string 216 | Args []string 217 | Dir string 218 | } 219 | 220 | type Case struct { 221 | SourceLocator hcl.Expression `hcl:"__source_locator,attr"` 222 | 223 | Name string `hcl:"name,label"` 224 | 225 | Args map[string]hcl.Expression `hcl:",remain"` 226 | } 227 | 228 | type App struct { 229 | BinName string 230 | 231 | Files map[string]*hcl.File 232 | Config *HCL2Config 233 | JobByName map[string]JobSpec 234 | 235 | Stdout, Stderr io.Writer 236 | 237 | Trace string 238 | 239 | sourceClient *source.Client 240 | 241 | initMu sync.Mutex 242 | 243 | Funcs map[string]function.Function 244 | 245 | JobLocalFuncs map[string]map[string]function.Function 246 | } 247 | -------------------------------------------------------------------------------- /pkg/app/values.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/hashicorp/hcl/v2" 7 | "github.com/zclconf/go-cty/cty" 8 | ) 9 | 10 | type Arg struct { 11 | name string 12 | 13 | typeExpr hcl.Expression 14 | 15 | defaultExpr hcl.Expression 16 | 17 | desc *string 18 | } 19 | 20 | func setValues(subject string, args map[string]cty.Value, ctx cty.Value, as []Arg, given map[string]interface{}, f SetOptsFunc) error { 21 | var pendingInputs []PendingInput 22 | 23 | for _, arg := range as { 24 | v, tpe, err := getValueFor(ctx, arg.name, arg.typeExpr, arg.defaultExpr, given) 25 | if err != nil { 26 | return fmt.Errorf("%s %q: %w", subject, arg.name, err) 27 | } 28 | 29 | if v == nil { 30 | if f != nil { 31 | pendingInputs = append(pendingInputs, PendingInput{Name: arg.name, Description: arg.desc, Type: *tpe}) 32 | } else { 33 | return fmt.Errorf("%s %q: missing value", subject, arg.name) 34 | } 35 | 36 | continue 37 | } 38 | 39 | args[arg.name] = *v 40 | } 41 | 42 | if len(pendingInputs) > 0 { 43 | if err := f(args, pendingInputs); err != nil { 44 | return fmt.Errorf("fulfilling missing %s from user input: %w", subject, err) 45 | } 46 | } 47 | 48 | return nil 49 | } 50 | 51 | func setParameterValues(subject string, ctx cty.Value, specs []Parameter, overrides map[string]interface{}) (map[string]cty.Value, error) { 52 | values := map[string]cty.Value{} 53 | 54 | { 55 | var args []Arg 56 | 57 | for _, p := range specs { 58 | args = append(args, Arg{ 59 | name: p.Name, 60 | desc: p.Description, 61 | typeExpr: p.Type, 62 | defaultExpr: p.Default, 63 | }) 64 | } 65 | 66 | if err := setValues(subject, values, ctx, args, overrides, nil); err != nil { 67 | return nil, err 68 | } 69 | } 70 | 71 | return values, nil 72 | } 73 | 74 | func setOptionValues(subject string, ctx cty.Value, specs []OptionSpec, overrides map[string]interface{}, f SetOptsFunc) (map[string]cty.Value, error) { 75 | values := map[string]cty.Value{} 76 | 77 | { 78 | var args []Arg 79 | 80 | for _, p := range specs { 81 | args = append(args, Arg{ 82 | name: p.Name, 83 | desc: p.Description, 84 | typeExpr: p.Type, 85 | defaultExpr: p.Default, 86 | }) 87 | } 88 | 89 | if err := setValues(subject, values, ctx, args, overrides, f); err != nil { 90 | return nil, err 91 | } 92 | } 93 | 94 | return values, nil 95 | } 96 | -------------------------------------------------------------------------------- /pkg/cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | 7 | variant "github.com/mumoshu/variant2" 8 | ) 9 | 10 | func main() { 11 | err := variant.RunMain(variant.Env{ 12 | Args: os.Args, 13 | Getenv: os.Getenv, 14 | Getwd: os.Getwd, 15 | }) 16 | 17 | var verr variant.Error 18 | 19 | var code int 20 | 21 | if err != nil { 22 | if ok := errors.As(err, &verr); ok { 23 | code = verr.ExitCode 24 | } else { 25 | code = 1 26 | } 27 | } else { 28 | code = 0 29 | } 30 | 31 | os.Exit(code) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/conf/funcs.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "github.com/hashicorp/hcl/v2/ext/tryfunc" 5 | "github.com/hashicorp/hcl/v2/ext/typeexpr" 6 | "github.com/hashicorp/terraform/lang/funcs" 7 | ctyyaml "github.com/zclconf/go-cty-yaml" 8 | "github.com/zclconf/go-cty/cty" 9 | "github.com/zclconf/go-cty/cty/function" 10 | "github.com/zclconf/go-cty/cty/function/stdlib" 11 | ) 12 | 13 | // Functions is a set of functions that are available in .variant files. 14 | // This set should cover all the functions available in moderately up-to-date version of Terraform, 15 | // so that the experience of using functions should be easier for users who are used with Terraform. 16 | func Functions(baseDir string) map[string]function.Function { 17 | return map[string]function.Function{ 18 | "abs": stdlib.AbsoluteFunc, 19 | "abspath": funcs.AbsPathFunc, 20 | "basename": funcs.BasenameFunc, 21 | "base64decode": funcs.Base64DecodeFunc, 22 | "base64encode": funcs.Base64EncodeFunc, 23 | "base64gzip": funcs.Base64GzipFunc, 24 | "base64sha256": funcs.Base64Sha256Func, 25 | "base64sha512": funcs.Base64Sha512Func, 26 | "bcrypt": funcs.BcryptFunc, 27 | "ceil": stdlib.CeilFunc, 28 | "chomp": stdlib.ChompFunc, 29 | "cidrhost": funcs.CidrHostFunc, 30 | "cidrnetmask": funcs.CidrNetmaskFunc, 31 | "cidrsubnet": funcs.CidrSubnetFunc, 32 | "cidrsubnets": funcs.CidrSubnetsFunc, 33 | "coalesce": stdlib.CoalesceFunc, 34 | "coalescelist": stdlib.CoalesceListFunc, 35 | "compact": stdlib.CompactFunc, 36 | "concat": stdlib.ConcatFunc, 37 | "contains": stdlib.ContainsFunc, 38 | "csvdecode": stdlib.CSVDecodeFunc, 39 | "dirname": funcs.DirnameFunc, 40 | "distinct": stdlib.DistinctFunc, 41 | "element": stdlib.ElementFunc, 42 | "chunklist": stdlib.ChunklistFunc, 43 | "file": funcs.MakeFileFunc(baseDir, false), 44 | "fileexists": funcs.MakeFileExistsFunc(baseDir), 45 | "fileset": funcs.MakeFileSetFunc(baseDir), 46 | "filebase64": funcs.MakeFileFunc(baseDir, true), 47 | "filebase64sha256": funcs.MakeFileBase64Sha256Func(baseDir), 48 | "filebase64sha512": funcs.MakeFileBase64Sha512Func(baseDir), 49 | "filemd5": funcs.MakeFileMd5Func(baseDir), 50 | "filesha1": funcs.MakeFileSha1Func(baseDir), 51 | "filesha256": funcs.MakeFileSha256Func(baseDir), 52 | "filesha512": funcs.MakeFileSha512Func(baseDir), 53 | "flatten": stdlib.FlattenFunc, 54 | "floor": stdlib.FloorFunc, 55 | "format": stdlib.FormatFunc, 56 | "formatdate": stdlib.FormatDateFunc, 57 | "formatlist": stdlib.FormatListFunc, 58 | "indent": stdlib.IndentFunc, 59 | "index": funcs.IndexFunc, 60 | "join": stdlib.JoinFunc, 61 | "jsondecode": stdlib.JSONDecodeFunc, 62 | "jsonencode": stdlib.JSONEncodeFunc, 63 | "keys": stdlib.KeysFunc, 64 | "length": funcs.LengthFunc, 65 | "list": funcs.ListFunc, 66 | "log": funcs.LogFunc, 67 | "lookup": funcs.LookupFunc, 68 | "lower": stdlib.LowerFunc, 69 | "map": funcs.MapFunc, 70 | "matchkeys": funcs.MatchkeysFunc, 71 | "max": stdlib.MaxFunc, 72 | "md5": funcs.Md5Func, 73 | "merge": stdlib.MergeFunc, 74 | "min": stdlib.MinFunc, 75 | "parseint": funcs.ParseIntFunc, 76 | "pathexpand": funcs.PathExpandFunc, 77 | "pow": funcs.PowFunc, 78 | "range": stdlib.RangeFunc, 79 | "regex": stdlib.RegexFunc, 80 | "regexall": stdlib.RegexAllFunc, 81 | "replace": funcs.ReplaceFunc, 82 | "reverse": stdlib.ReverseFunc, 83 | "rsadecrypt": funcs.RsaDecryptFunc, 84 | "setintersection": stdlib.SetIntersectionFunc, 85 | "setproduct": stdlib.SetProductFunc, 86 | "setunion": stdlib.SetUnionFunc, 87 | "sha1": funcs.Sha1Func, 88 | "sha256": funcs.Sha256Func, 89 | "sha512": funcs.Sha512Func, 90 | "signum": funcs.SignumFunc, 91 | "slice": stdlib.SliceFunc, 92 | "sort": stdlib.SortFunc, 93 | "split": stdlib.SplitFunc, 94 | "strrev": stdlib.ReverseFunc, 95 | "substr": stdlib.SubstrFunc, 96 | "timestamp": funcs.TimestampFunc, 97 | "timeadd": funcs.TimeAddFunc, 98 | "title": stdlib.TitleFunc, 99 | "tostring": funcs.MakeToFunc(cty.String), 100 | "tonumber": funcs.MakeToFunc(cty.Number), 101 | "tobool": funcs.MakeToFunc(cty.Bool), 102 | "toset": funcs.MakeToFunc(cty.Set(cty.DynamicPseudoType)), 103 | "tolist": funcs.MakeToFunc(cty.List(cty.DynamicPseudoType)), 104 | "tomap": funcs.MakeToFunc(cty.Map(cty.DynamicPseudoType)), 105 | "transpose": funcs.TransposeFunc, 106 | "trim": stdlib.TrimFunc, 107 | "trimprefix": stdlib.TrimPrefixFunc, 108 | "trimspace": stdlib.TrimSpaceFunc, 109 | "trimsuffix": stdlib.TrimSuffixFunc, 110 | "upper": stdlib.UpperFunc, 111 | "urlencode": funcs.URLEncodeFunc, 112 | "uuid": funcs.UUIDFunc, 113 | "uuidv5": funcs.UUIDV5Func, 114 | "values": stdlib.ValuesFunc, 115 | "yamldecode": ctyyaml.YAMLDecodeFunc, 116 | "yamlencode": ctyyaml.YAMLEncodeFunc, 117 | "zipmap": stdlib.ZipmapFunc, 118 | "try": tryfunc.TryFunc, 119 | "can": tryfunc.CanFunc, 120 | "convert": typeexpr.ConvertFunc, 121 | "jsonpath": JSONPathFunc, 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /pkg/conf/jsonpath.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/tidwall/gjson" 7 | "github.com/zclconf/go-cty/cty" 8 | "github.com/zclconf/go-cty/cty/function" 9 | ctyjson "github.com/zclconf/go-cty/cty/json" 10 | "golang.org/x/xerrors" 11 | ) 12 | 13 | func getValueAtJSONPath(data, path string) (cty.Value, error) { 14 | result := gjson.Get(data, path) 15 | if !result.Exists() { 16 | return cty.NullVal(cty.String), fmt.Errorf("no value found at jsonpath %q: not found", path) 17 | } 18 | 19 | raw := []byte(result.Raw) 20 | 21 | ty, err := ctyjson.ImpliedType(raw) 22 | if err != nil { 23 | return cty.DynamicVal, xerrors.Errorf("determining implied type of %s: %w", string(raw), err) 24 | } 25 | 26 | return ctyjson.Unmarshal(raw, ty) 27 | } 28 | 29 | // JSONPathFunc takes JSON and a query to fetch the value for the query. 30 | var JSONPathFunc = function.New(&function.Spec{ 31 | Params: []function.Parameter{ 32 | { 33 | Name: "data", 34 | Type: cty.String, 35 | }, 36 | { 37 | Name: "query", 38 | Type: cty.String, 39 | }, 40 | }, 41 | VarParam: &function.Parameter{ 42 | Name: "file", 43 | Type: cty.String, 44 | }, 45 | Type: func(args []cty.Value) (cty.Type, error) { 46 | data := args[0].AsString() 47 | query := args[1].AsString() 48 | 49 | v, err := getValueAtJSONPath(data, query) 50 | 51 | return v.Type(), err 52 | }, 53 | Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { 54 | data := args[0].AsString() 55 | query := args[1].AsString() 56 | 57 | v, err := getValueAtJSONPath(data, query) 58 | 59 | return v, err 60 | }, 61 | }) 62 | -------------------------------------------------------------------------------- /pkg/conf/load.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "path/filepath" 5 | 6 | "golang.org/x/xerrors" 7 | 8 | "github.com/mumoshu/variant2/pkg/fs" 9 | ) 10 | 11 | const ( 12 | VariantFileExt = ".variant" 13 | ) 14 | 15 | // FindVariantFiles walks the given path and returns the files ending whose ext is .variant 16 | // Also, it returns the path if the path is just a file and a HCL file. 17 | func FindVariantFiles(fs *fs.FileSystem, path string) ([]string, error) { 18 | var ( 19 | files []string 20 | err error 21 | ) 22 | 23 | fi, err := fs.Stat(path) 24 | if err != nil { 25 | return files, xerrors.Errorf("stat: %w", err) 26 | } 27 | 28 | if fi.IsDir() { 29 | variantFilesPattern := filepath.Join(path, "*"+VariantFileExt+"*") 30 | 31 | found, err := fs.Glob(variantFilesPattern) 32 | if err != nil { 33 | return nil, xerrors.Errorf("glob %q: %w", variantFilesPattern, err) 34 | } 35 | 36 | for _, f := range found { 37 | switch filepath.Ext(f) { 38 | case VariantFileExt, ".json": 39 | default: 40 | continue 41 | } 42 | 43 | info, err := fs.Stat(f) 44 | if err != nil { 45 | return nil, xerrors.Errorf("stat %s: %w", f, err) 46 | } 47 | 48 | if info.IsDir() { 49 | continue 50 | } 51 | 52 | files = append(files, f) 53 | } 54 | 55 | return files, nil 56 | } 57 | 58 | switch filepath.Ext(path) { 59 | case VariantFileExt, ".json": 60 | files = append(files, path) 61 | } 62 | 63 | return files, xerrors.Errorf("stat %s: %w", path, err) 64 | } 65 | -------------------------------------------------------------------------------- /pkg/controller/api.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 5 | "k8s.io/apimachinery/pkg/runtime/schema" 6 | ) 7 | 8 | const ( 9 | coreGroup = "core.variant.run" 10 | coreVersion = "v1beta1" 11 | ) 12 | 13 | var reconciliationGroupVersionKind = schema.GroupVersionKind{ 14 | Group: coreGroup, 15 | Version: coreVersion, 16 | Kind: "Reconciliation", 17 | } 18 | 19 | func newReconciliation() *unstructured.Unstructured { 20 | obj := &unstructured.Unstructured{} 21 | 22 | obj.SetGroupVersionKind(reconciliationGroupVersionKind) 23 | 24 | return obj 25 | } 26 | -------------------------------------------------------------------------------- /pkg/controller/capture.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os" 7 | 8 | "golang.org/x/sync/errgroup" 9 | "golang.org/x/xerrors" 10 | ) 11 | 12 | func CaptureOutput(f func(io.Writer, io.Writer) error) (string, error) { 13 | buf := &bytes.Buffer{} 14 | 15 | outRead, outWrite := io.Pipe() 16 | outBuf := io.TeeReader(outRead, buf) 17 | 18 | errRead, errWrite := io.Pipe() 19 | errBuf := io.TeeReader(errRead, buf) 20 | 21 | eg := &errgroup.Group{} 22 | 23 | eg.Go(func() error { 24 | if _, err := io.Copy(os.Stdout, outBuf); err != nil { 25 | return xerrors.Errorf("copying to stdout: %w", err) 26 | } 27 | 28 | return nil 29 | }) 30 | 31 | eg.Go(func() error { 32 | if _, err := io.Copy(os.Stderr, errBuf); err != nil { 33 | return xerrors.Errorf("copying to stderr: %w", err) 34 | } 35 | 36 | return nil 37 | }) 38 | 39 | err := f(outWrite, errWrite) 40 | 41 | outWrite.Close() 42 | errWrite.Close() 43 | 44 | if egErr := eg.Wait(); egErr != nil { 45 | panic(egErr) 46 | } 47 | 48 | if err != nil { 49 | return "", xerrors.Errorf("running command: %w", err) 50 | } 51 | 52 | return buf.String(), nil 53 | } 54 | -------------------------------------------------------------------------------- /pkg/controller/config.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | type Config struct { 10 | controllerName string 11 | resyncPeriod string 12 | forKind string 13 | group string 14 | version string 15 | jobOnApply string 16 | jobOnDestroy string 17 | } 18 | 19 | func getConfigFromEnv() (*Config, error) { 20 | getEnv := func(n string) (string, string) { 21 | name := EnvPrefix + n 22 | value := os.Getenv(name) 23 | 24 | return name, value 25 | } 26 | 27 | controllerNameEnv, controllerName := getEnv("NAME") 28 | if controllerName == "" { 29 | return nil, fmt.Errorf("missing required environment variable: %s", controllerNameEnv) 30 | } 31 | 32 | _, forAPIVersion := getEnv("FOR_API_VERSION") 33 | if forAPIVersion == "" { 34 | forAPIVersion = coreGroup + "/" + coreVersion 35 | } 36 | 37 | _, forKind := getEnv("FOR_KIND") 38 | if forKind == "" { 39 | forKind = "Resource" 40 | } 41 | 42 | _, resyncPeriod := getEnv("RESYNC_PERIOD") 43 | 44 | groupVersion := strings.Split(forAPIVersion, "/") 45 | group := groupVersion[0] 46 | version := groupVersion[1] 47 | 48 | jobOnApplyEnv, jobOnApply := getEnv("JOB_ON_APPLY") 49 | if jobOnApply == "" { 50 | return nil, fmt.Errorf("missing required environment variable: %s", jobOnApplyEnv) 51 | } 52 | 53 | jobOnDestroyEnv, jobOnDestroy := getEnv("JOB_ON_DESTROY") 54 | if jobOnDestroy == "" { 55 | return nil, fmt.Errorf("missing required environment variable: %s", jobOnDestroyEnv) 56 | } 57 | 58 | return &Config{ 59 | controllerName: controllerName, 60 | resyncPeriod: resyncPeriod, 61 | forKind: forKind, 62 | group: group, 63 | version: version, 64 | jobOnApply: jobOnApply, 65 | jobOnDestroy: jobOnDestroy, 66 | }, nil 67 | } 68 | -------------------------------------------------------------------------------- /pkg/controller/controller.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/go-logr/logr" 9 | "golang.org/x/xerrors" 10 | "k8s.io/apimachinery/pkg/api/errors" 11 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 12 | "sigs.k8s.io/controller-runtime/pkg/client" 13 | ) 14 | 15 | type controller struct { 16 | // controllerName is the name of this controller that is shown in the logs 17 | controllerName string 18 | 19 | // podName is the hostname of where the controller is running on. 20 | // Stored in Reconciliation objects so that the operator can track which controller in which pod has done the 21 | // reconciliation. 22 | podName string 23 | 24 | // Kubernetes client to be used for querying target objects and managing Reconciliation objects 25 | runtimeClient client.Client 26 | 27 | // Runs `variant run ` and returns combined output and/or error 28 | run func([]string) (string, error) 29 | 30 | log logr.Logger 31 | } 32 | 33 | func (c *controller) do(job string, obj *unstructured.Unstructured) error { 34 | args := strings.Split(job, " ") 35 | 36 | m, found, err := unstructured.NestedMap(obj.Object, "spec") 37 | if !found { 38 | return fmt.Errorf(`"spec" field not found: %v`, obj.Object) 39 | } 40 | 41 | if err != nil { 42 | return xerrors.Errorf("getting nested map from the object: %w", err) 43 | } 44 | 45 | for k, v := range m { 46 | args = append(args, "--"+k, fmt.Sprintf("%v", v)) 47 | } 48 | 49 | c.log.Info("Running Variant", "args", strings.Join(args, " ")) 50 | 51 | args = append([]string{"run"}, args...) 52 | 53 | combinedLogs, err := c.run(args) 54 | if err != nil { 55 | return xerrors.Errorf("executing %v: %w", args, err) 56 | } 57 | 58 | if err := c.logReconciliation(obj, job, combinedLogs); err != nil { 59 | return xerrors.Errorf("logging result of %q: %w", job, err) 60 | } 61 | 62 | return nil 63 | } 64 | 65 | func (c *controller) logReconciliation(orig *unstructured.Unstructured, job, combinedLogs string) error { 66 | name := orig.GetName() 67 | namespace := orig.GetNamespace() 68 | 69 | st := &unstructured.Unstructured{} 70 | st.SetGroupVersionKind(orig.GroupVersionKind()) 71 | 72 | if err := c.runtimeClient.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: name}, st); err != nil { 73 | return fmt.Errorf("getting object %q: %w", name, err) 74 | } 75 | 76 | gen, ok, err := unstructured.NestedInt64(st.Object, "metadata", "generation") 77 | if !ok { 78 | return fmt.Errorf("missing Resource.Generation: %w", err) 79 | } 80 | 81 | if err != nil { 82 | return xerrors.Errorf("getting metadata.generation from %s: %w", name, err) 83 | } 84 | 85 | reconName := name + "-" + fmt.Sprintf("%d", gen) 86 | 87 | obj := newReconciliation() 88 | 89 | var update bool 90 | 91 | getErr := c.runtimeClient.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: reconName}, obj) 92 | if getErr != nil { 93 | if !errors.IsNotFound(getErr) { 94 | return fmt.Errorf("getting reconciliation object: %w", err) 95 | } 96 | } else { 97 | update = true 98 | } 99 | 100 | // Use of GenerateName results in 404 101 | // obj.SetGenerateName(name + "-") 102 | obj.SetName(reconName) 103 | // Missing Namespace results in 404 104 | obj.SetNamespace(namespace) 105 | obj.SetLabels(map[string]string{ 106 | "core.variant.run/event": "apply", 107 | "core.variant.run/controller": c.controllerName, 108 | "core.variant.run/pod": c.podName, 109 | }) 110 | 111 | spec, ok, err := unstructured.NestedMap(st.Object, "spec") 112 | if !ok { 113 | return fmt.Errorf("missing Resource.Spec: %w", err) 114 | } 115 | 116 | if err != nil { 117 | return xerrors.Errorf("calling unstructured.NestedMap: %w", err) 118 | } 119 | 120 | if err := unstructured.SetNestedField(obj.Object, job, "spec", "job"); err != nil { 121 | return xerrors.Errorf("setting nested field spec.job: %w", err) 122 | } 123 | 124 | if err := unstructured.SetNestedMap(obj.Object, spec, "spec", "resource"); err != nil { 125 | return xerrors.Errorf("setting nested map spec.resource: %w", err) 126 | } 127 | 128 | if err := unstructured.SetNestedField(obj.Object, combinedLogs, "spec", "combinedLogs", "data"); err != nil { 129 | return xerrors.Errorf("setting nested field spec.combinedLogs.data: %w", err) 130 | } 131 | 132 | if update { 133 | if err := c.runtimeClient.Update(context.TODO(), obj); err != nil { 134 | return fmt.Errorf("updating reconciliation object: %w", err) 135 | } 136 | } else { 137 | if err := c.runtimeClient.Create(context.TODO(), obj); err != nil { 138 | return fmt.Errorf("creating reconciliation object: %w", err) 139 | } 140 | } 141 | 142 | return nil 143 | } 144 | -------------------------------------------------------------------------------- /pkg/controller/handler.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "github.com/summerwind/whitebox-controller/handler" 5 | "github.com/summerwind/whitebox-controller/reconciler/state" 6 | ) 7 | 8 | func StateHandlerFunc(f func(*state.State) error) handler.StateHandler { 9 | return &stateHandler{ 10 | f: f, 11 | } 12 | } 13 | 14 | type stateHandler struct { 15 | f func(*state.State) error 16 | } 17 | 18 | func (h stateHandler) HandleState(s *state.State) error { 19 | return h.f(s) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/controller/run.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | 7 | "github.com/summerwind/whitebox-controller/config" 8 | "github.com/summerwind/whitebox-controller/manager" 9 | "github.com/summerwind/whitebox-controller/reconciler/state" 10 | "golang.org/x/xerrors" 11 | "k8s.io/apimachinery/pkg/runtime/schema" 12 | 13 | // We import these here rather than in main to automate setting up cloud-provider-specific authentication strategies. 14 | _ "k8s.io/client-go/plugin/pkg/client/auth/azure" 15 | 16 | // We import these here rather than in main to automate setting up cloud-provider-specific authentication strategies. 17 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 18 | _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" 19 | 20 | // We import these here rather than in main to automate setting up cloud-provider-specific authentication strategies. 21 | kconfig "sigs.k8s.io/controller-runtime/pkg/client/config" 22 | logf "sigs.k8s.io/controller-runtime/pkg/log" 23 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 24 | "sigs.k8s.io/controller-runtime/pkg/manager/signals" 25 | ) 26 | 27 | const ( 28 | EnvPrefix = "VARIANT_CONTROLLER_" 29 | ) 30 | 31 | func RunRequested() bool { 32 | for _, env := range os.Environ() { 33 | if strings.HasPrefix(env, EnvPrefix) { 34 | return true 35 | } 36 | } 37 | 38 | return false 39 | } 40 | 41 | func Run(run func([]string) (string, error)) (finalErr error) { 42 | logf.SetLogger(zap.New()) 43 | 44 | defer func() { 45 | if finalErr != nil { 46 | logf.Log.Error(finalErr, "Error while running controller") 47 | } 48 | }() 49 | 50 | kc, err := kconfig.GetConfig() 51 | if err != nil { 52 | return xerrors.Errorf("getting kubernetes client config: %w", err) 53 | } 54 | 55 | conf, err := getConfigFromEnv() 56 | if err != nil { 57 | return xerrors.Errorf("getting config from envvars: %w", err) 58 | } 59 | 60 | podName, err := os.Hostname() 61 | if err != nil { 62 | return xerrors.Errorf("getting pod name from hostname: %w", err) 63 | } 64 | 65 | ctl := &controller{ 66 | log: logf.Log.WithName(conf.controllerName), 67 | runtimeClient: nil, 68 | run: run, 69 | podName: podName, 70 | controllerName: conf.controllerName, 71 | } 72 | 73 | handle := func(st *state.State, job string) (finalErr error) { 74 | return ctl.do(job, st.Object) 75 | } 76 | 77 | applyHandler := StateHandlerFunc(func(st *state.State) error { 78 | return handle(st, conf.jobOnApply) 79 | }) 80 | 81 | destroyHandler := StateHandlerFunc(func(st *state.State) error { 82 | return handle(st, conf.jobOnDestroy) 83 | }) 84 | 85 | whiteboxConfig := &config.Config{ 86 | Name: conf.controllerName, 87 | Resources: []*config.ResourceConfig{ 88 | { 89 | GroupVersionKind: schema.GroupVersionKind{ 90 | Group: conf.group, 91 | Version: conf.version, 92 | Kind: conf.forKind, 93 | }, 94 | Reconciler: &config.ReconcilerConfig{ 95 | HandlerConfig: config.HandlerConfig{ 96 | StateHandler: applyHandler, 97 | }, 98 | }, 99 | Finalizer: &config.HandlerConfig{ 100 | StateHandler: destroyHandler, 101 | }, 102 | ResyncPeriod: conf.resyncPeriod, 103 | }, 104 | }, 105 | Webhook: nil, 106 | } 107 | 108 | mgr, err := manager.New(whiteboxConfig, kc) 109 | if err != nil { 110 | return xerrors.Errorf("creating controller-manager: %w", err) 111 | } 112 | 113 | ctl.runtimeClient = mgr.GetClient() 114 | 115 | err = mgr.Start(signals.SetupSignalHandler()) 116 | if err != nil { 117 | return xerrors.Errorf("starting controller-manager: %w", err) 118 | } 119 | 120 | return nil 121 | } 122 | -------------------------------------------------------------------------------- /pkg/fs/fs.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | "sort" 11 | "sync" 12 | 13 | "github.com/rakyll/statik/fs" 14 | "golang.org/x/xerrors" 15 | ) 16 | 17 | const ( 18 | VendorPrefix = "vendored" 19 | ) 20 | 21 | type FileSystem struct { 22 | sync.Once 23 | fs http.FileSystem 24 | } 25 | 26 | type noopFS struct { 27 | } 28 | 29 | func (f *noopFS) Open(_ string) (http.File, error) { 30 | return nil, os.ErrNotExist 31 | } 32 | 33 | func (s *FileSystem) ReadFile(path string) ([]byte, error) { 34 | fs, err := s.getFS() 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | f, err := fs.Open(s.vendored(path)) 40 | if errors.Is(err, os.ErrNotExist) { 41 | return ioutil.ReadFile(path) 42 | } 43 | defer f.Close() 44 | 45 | bs, err := ioutil.ReadAll(f) 46 | if err != nil { 47 | return nil, fmt.Errorf("reading statik file: %w", err) 48 | } 49 | 50 | return bs, nil 51 | } 52 | 53 | func (s *FileSystem) Stat(path string) (os.FileInfo, error) { 54 | fs, err := s.getFS() 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | f, err := fs.Open(s.vendored(path)) 60 | if errors.Is(err, os.ErrNotExist) { 61 | return os.Stat(path) 62 | } 63 | defer f.Close() 64 | 65 | return f.Stat() 66 | } 67 | 68 | func (s *FileSystem) Glob(pattern string) ([]string, error) { 69 | fs, err := s.getFS() 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | dir, _ := filepath.Split(s.vendored(pattern)) 75 | 76 | found, err := glob(fs, dir, s.vendored(pattern)) 77 | if err != nil { 78 | return nil, fmt.Errorf("glob using statik: %w", err) 79 | } 80 | 81 | if len(found) > 0 { 82 | return found, nil 83 | } 84 | 85 | found, err = filepath.Glob(pattern) 86 | if err != nil { 87 | return nil, fmt.Errorf("glob using filepath: %w", err) 88 | } 89 | 90 | return found, nil 91 | } 92 | 93 | func (s *FileSystem) getFS() (http.FileSystem, error) { 94 | var err error 95 | 96 | s.Once.Do(func() { 97 | s.fs, err = fs.New() 98 | if err != nil { 99 | s.fs = &noopFS{} 100 | err = nil 101 | } 102 | }) 103 | 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | return s.fs, nil 109 | } 110 | 111 | func (s *FileSystem) vendored(path string) string { 112 | return filepath.Join(string(filepath.Separator), path) 113 | } 114 | 115 | func glob(fs http.FileSystem, dir, pattern string) ([]string, error) { 116 | d, err := fs.Open(dir) 117 | if err != nil { 118 | return nil, nil 119 | } 120 | defer d.Close() 121 | 122 | fi, err := d.Stat() 123 | if err != nil { 124 | return nil, nil 125 | } 126 | 127 | if !fi.IsDir() { 128 | return nil, nil 129 | } 130 | 131 | entries, err := d.Readdir(-1) 132 | if err != nil { 133 | return nil, fmt.Errorf("readdir: %w", err) 134 | } 135 | 136 | var names []string 137 | 138 | for _, ent := range entries { 139 | if ent.IsDir() { 140 | subEntries, err := glob(fs, "/"+ent.Name(), pattern) 141 | if err != nil { 142 | return nil, err 143 | } 144 | 145 | names = append(names, subEntries...) 146 | } else { 147 | names = append(names, filepath.Join(dir, ent.Name())) 148 | } 149 | } 150 | 151 | sort.Strings(names) 152 | 153 | var m []string 154 | 155 | for _, n := range names { 156 | matched, err := filepath.Match(pattern, n) 157 | if err != nil { 158 | return m, xerrors.Errorf("matching pattern %s against %s: %w", pattern, n, err) 159 | } 160 | 161 | if matched { 162 | m = append(m, n) 163 | } 164 | } 165 | 166 | return m, nil 167 | } 168 | -------------------------------------------------------------------------------- /pkg/kube/client.go: -------------------------------------------------------------------------------- 1 | package kube 2 | 3 | import ( 4 | sourcev1beta1 "github.com/fluxcd/source-controller/api/v1beta1" 5 | "golang.org/x/xerrors" 6 | "k8s.io/apimachinery/pkg/runtime" 7 | "sigs.k8s.io/controller-runtime/pkg/client" 8 | ) 9 | 10 | func NewClient() (client.Client, error) { 11 | cfg, err := NewRestConfig() 12 | if err != nil { 13 | return nil, err 14 | } 15 | 16 | scheme := runtime.NewScheme() 17 | 18 | if err := sourcev1beta1.AddToScheme(scheme); err != nil { 19 | return nil, xerrors.Errorf("adding sourcev1beta1: %w", err) 20 | } 21 | 22 | client, err := client.New(cfg, client.Options{Scheme: scheme}) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | return client, nil 28 | } 29 | -------------------------------------------------------------------------------- /pkg/kube/restconfig.go: -------------------------------------------------------------------------------- 1 | package kube 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "path/filepath" 7 | 8 | "golang.org/x/xerrors" 9 | "k8s.io/client-go/rest" 10 | "k8s.io/client-go/tools/clientcmd" 11 | "k8s.io/client-go/util/homedir" 12 | ) 13 | 14 | func NewRestConfig() (*rest.Config, error) { 15 | var kubeconfig string 16 | 17 | kubeconfig, ok := os.LookupEnv("KUBECONFIG") 18 | if !ok { 19 | kubeconfig = filepath.Join(homedir.HomeDir(), ".kube", "config") 20 | } 21 | 22 | var config *rest.Config 23 | 24 | if info, _ := os.Stat(kubeconfig); info == nil { 25 | var err error 26 | 27 | log.Printf("Using in-cluster Kubernetes API client") 28 | 29 | config, err = rest.InClusterConfig() 30 | if err != nil { 31 | return nil, xerrors.Errorf("getting in-cluster config: %w", err) 32 | } 33 | } else { 34 | var err error 35 | 36 | log.Printf("Using kubeconfig-based Kubernetes API client") 37 | 38 | config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) 39 | if err != nil { 40 | return nil, xerrors.Errorf("building rest config: %w", err) 41 | } 42 | } 43 | 44 | return config, nil 45 | } 46 | -------------------------------------------------------------------------------- /pkg/sdk/vars.go: -------------------------------------------------------------------------------- 1 | package sdk 2 | 3 | var Version string 4 | 5 | var ModReplaces string 6 | -------------------------------------------------------------------------------- /pkg/slack/README.md: -------------------------------------------------------------------------------- 1 | # Slackbot integration for Variant 2 2 | 3 | This package provides a HTTP server that handles Slack interaction callbacks so that 4 | it can run any variant command in response to a Slack slash command. 5 | 6 | ## Acknowledgements 7 | 8 | I've learned how to code a Slack bot in Go by reading the following repositories and articles: 9 | 10 | - https://github.com/kpurdon/slappd 11 | - https://github.com/shiimaxx/slack-coffeebot 12 | - https://github.com/eure/cafe-bot/ 13 | -------------------------------------------------------------------------------- /pkg/slack/http.go: -------------------------------------------------------------------------------- 1 | package slack 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | type HTTPHandler interface { 8 | ServeHTTP(w http.ResponseWriter, r *http.Request) 9 | } 10 | -------------------------------------------------------------------------------- /pkg/slack/interaction.go: -------------------------------------------------------------------------------- 1 | package slack 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "net/url" 10 | 11 | "github.com/nlopes/slack" 12 | ) 13 | 14 | type InteractionsHandler func(callback slack.InteractionCallback) (interface{}, error) 15 | 16 | type interactionsHTTPHandler struct { 17 | VerificationToken string 18 | 19 | handler InteractionsHandler 20 | } 21 | 22 | func (h interactionsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 23 | if r.Method != http.MethodPost { 24 | log.Printf("[ERROR] Invalid method: %s", r.Method) 25 | w.WriteHeader(http.StatusMethodNotAllowed) 26 | 27 | return 28 | } 29 | 30 | buf, err := ioutil.ReadAll(r.Body) 31 | if err != nil { 32 | log.Printf("[ERROR] Failed to read request body: %s", err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | 35 | return 36 | } 37 | 38 | jsonStr, err := url.QueryUnescape(string(buf)[8:]) 39 | if err != nil { 40 | log.Printf("[ERROR] Failed to unespace request body: %s", err) 41 | w.WriteHeader(http.StatusInternalServerError) 42 | 43 | return 44 | } 45 | 46 | var message slack.InteractionCallback 47 | if err := json.Unmarshal([]byte(jsonStr), &message); err != nil { 48 | log.Printf("[ERROR] Failed to decode json message from slack: %s", jsonStr) 49 | w.WriteHeader(http.StatusInternalServerError) 50 | 51 | return 52 | } 53 | 54 | // Only accept message from slack with valid token 55 | if message.Token != h.VerificationToken { 56 | log.Printf("[ERROR] Invalid token: %s", message.Token) 57 | w.WriteHeader(http.StatusUnauthorized) 58 | 59 | return 60 | } 61 | 62 | output, err := h.handler(message) 63 | if err != nil { 64 | log.Printf("[ERROR] handler failed: %v", err) 65 | w.WriteHeader(http.StatusInternalServerError) 66 | 67 | return 68 | } 69 | 70 | w.Header().Add("Content-type", "application/json") 71 | w.WriteHeader(http.StatusOK) 72 | 73 | if output == nil { 74 | fmt.Fprintf(w, "") 75 | 76 | return 77 | } 78 | 79 | if err := json.NewEncoder(w).Encode(&output); err != nil { 80 | panic(err) 81 | } 82 | } 83 | 84 | func slackInteractionsToHTTPHandler(handler InteractionsHandler, handler2 func(cmd slack.SlashCommand) (interface{}, error), verificationToken string) HTTPHandler { 85 | mux := http.NewServeMux() 86 | mux.Handle("/interactions", interactionsHTTPHandler{ 87 | VerificationToken: verificationToken, 88 | handler: handler, 89 | }) 90 | mux.Handle("/slashcommands", slashCommandsHTTPHandler{ 91 | VerificationToken: verificationToken, 92 | handler: handler2, 93 | }) 94 | 95 | return mux 96 | } 97 | -------------------------------------------------------------------------------- /pkg/slack/slashcommand.go: -------------------------------------------------------------------------------- 1 | package slack 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/nlopes/slack" 9 | ) 10 | 11 | // See https://api.slack.com/interactivity/slash-commands 12 | type slashCommandsHTTPHandler struct { 13 | VerificationToken string 14 | 15 | handler func(callback slack.SlashCommand) (interface{}, error) 16 | } 17 | 18 | func (h slashCommandsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 19 | if r.Method != http.MethodPost { 20 | log.Printf("[ERROR] Invalid method: %s", r.Method) 21 | w.WriteHeader(http.StatusMethodNotAllowed) 22 | 23 | return 24 | } 25 | 26 | cmd, err := slack.SlashCommandParse(r) 27 | if err != nil { 28 | log.Printf("[ERROR] failed to parse slash command: %v", err) 29 | w.WriteHeader(http.StatusBadRequest) 30 | 31 | return 32 | } 33 | 34 | // Only accept message from slack with valid token 35 | if cmd.Token != h.VerificationToken { 36 | log.Printf("[ERROR] Invalid token: %s", cmd.Token) 37 | w.WriteHeader(http.StatusUnauthorized) 38 | 39 | return 40 | } 41 | 42 | output, err := h.handler(cmd) 43 | if err != nil { 44 | log.Printf("[ERROR] handler failed: %v", err) 45 | w.WriteHeader(http.StatusInternalServerError) 46 | 47 | return 48 | } 49 | 50 | w.Header().Add("Content-type", "application/json") 51 | w.WriteHeader(http.StatusOK) 52 | 53 | if err := json.NewEncoder(w).Encode(&output); err != nil { 54 | panic(err) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /pkg/source/src.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | "time" 11 | 12 | "github.com/fluxcd/pkg/untar" 13 | sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" 14 | "golang.org/x/xerrors" 15 | apierrors "k8s.io/apimachinery/pkg/api/errors" 16 | "k8s.io/apimachinery/pkg/types" 17 | "sigs.k8s.io/controller-runtime/pkg/client" 18 | ) 19 | 20 | type Client struct { 21 | Client client.Client 22 | } 23 | 24 | // FetchTarball returns an io.ReadCloser that contains the http response body on successful request. 25 | // It's the user's responsibility to close any non-nil ReadCloser otherwise the 26 | // original http.Response.Body leaks. 27 | func (c *Client) FetchTarball(ctx context.Context, url string) (io.ReadCloser, error) { 28 | req, err := http.NewRequest("GET", url, nil) 29 | if err != nil { 30 | return nil, fmt.Errorf("failed to create HTTP request for %s, error: %w", url, err) 31 | } 32 | 33 | //nolint:bodyclose 34 | resp, err := http.DefaultClient.Do(req.WithContext(ctx)) 35 | if err != nil { 36 | return nil, fmt.Errorf("failed to download artifact from %s, error: %w", url, err) 37 | } 38 | 39 | if resp.StatusCode != http.StatusOK { 40 | return nil, fmt.Errorf("artifact '%s' download failed (status code: %s)", url, resp.Status) 41 | } 42 | 43 | return resp.Body, nil 44 | } 45 | 46 | func (c *Client) DownloadTarball(tarballURL string, f io.Writer) error { 47 | body, err := c.FetchTarball(context.Background(), tarballURL) 48 | 49 | defer func() { 50 | if body != nil { 51 | _ = body.Close() 52 | } 53 | }() 54 | 55 | if err != nil { 56 | return xerrors.Errorf("fetching tarball: %w", err) 57 | } 58 | 59 | if _, err = io.Copy(f, body); err != nil { 60 | return err 61 | } 62 | 63 | return nil 64 | } 65 | 66 | func (c *Client) ExtractTarball(url, dir string) error { 67 | timeout := time.Second * 30 68 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 69 | 70 | defer cancel() 71 | 72 | body, err := c.FetchTarball(ctx, url) 73 | if err != nil { 74 | return xerrors.Errorf("fetching %s: %w", url, err) 75 | } 76 | 77 | defer func() { 78 | if body != nil { 79 | _ = body.Close() 80 | } 81 | }() 82 | 83 | if _, err = untar.Untar(body, dir); err != nil { 84 | return fmt.Errorf("faild to untar artifact, error: %w", err) 85 | } 86 | 87 | return nil 88 | } 89 | 90 | type TransientError struct { 91 | err error 92 | } 93 | 94 | func (e *TransientError) Error() string { 95 | return e.err.Error() 96 | } 97 | 98 | func (e *TransientError) Unwrap() error { 99 | return e.err 100 | } 101 | 102 | func (c *Client) ExtractSource(ctx context.Context, kind, ns, name string) (string, error) { 103 | // resolve source reference 104 | source, err := c.getSource(ctx, kind, ns, name) 105 | if err != nil { 106 | if apierrors.IsNotFound(err) { 107 | msg := "Source not found" 108 | // do not requeue on this error, when the artifact is created the watcher should trigger a reconciliation 109 | return "", xerrors.Errorf("getting artifact: %s: %w", msg, err) 110 | } 111 | 112 | // retry on transient errors 113 | return "", xerrors.Errorf("getting artifact: %w", &TransientError{err}) 114 | } 115 | 116 | if source.GetArtifact() == nil { 117 | msg := "Source is not ready, artifact not found" 118 | // do not requeue on this error, when the artifact is created the watcher should trigger a reconciliation 119 | return "", fmt.Errorf("getting artifact: %s", msg) 120 | } 121 | 122 | artifact := source.GetArtifact() 123 | tarballURL := artifact.URL 124 | 125 | var dir string 126 | 127 | if artifact.Revision != "" { 128 | dir = filepath.Join(os.TempDir(), "variant", "cache", "source", fmt.Sprintf("%s-%s-%s-%s", kind, ns, name, artifact.Revision)) 129 | } else { 130 | dir = filepath.Join(os.TempDir(), "variant", "cache", "source", fmt.Sprintf("%s-%s-%s", kind, ns, name)) 131 | } 132 | 133 | if err := os.MkdirAll(dir, 0o700); err != nil { 134 | return "", xerrors.Errorf("creating source cache dir: %w", err) 135 | } 136 | 137 | if info, err := os.Stat(dir); info == nil { 138 | return "", xerrors.Errorf("looking for source cache: %w", err) 139 | } 140 | 141 | extErr := c.ExtractTarball(tarballURL, dir) 142 | 143 | if extErr != nil { 144 | return "", xerrors.Errorf("extracting tarball: %w", extErr) 145 | } 146 | 147 | return dir, nil 148 | } 149 | 150 | func (c *Client) getSource(ctx context.Context, kind, namespace, name string) (sourcev1.Source, error) { 151 | var source sourcev1.Source 152 | 153 | namespacedName := types.NamespacedName{ 154 | Namespace: namespace, 155 | Name: name, 156 | } 157 | 158 | if kind == "" || kind == "git" { 159 | kind = sourcev1.GitRepositoryKind 160 | } 161 | 162 | switch kind { 163 | case sourcev1.GitRepositoryKind: 164 | var repository sourcev1.GitRepository 165 | 166 | err := c.Client.Get(ctx, namespacedName, &repository) 167 | if err != nil { 168 | if apierrors.IsNotFound(err) { 169 | return source, err 170 | } 171 | 172 | return source, fmt.Errorf("unable to get source '%s': %w", namespacedName, err) 173 | } 174 | 175 | source = &repository 176 | case sourcev1.BucketKind: 177 | var bucket sourcev1.Bucket 178 | 179 | err := c.Client.Get(ctx, namespacedName, &bucket) 180 | if err != nil { 181 | if apierrors.IsNotFound(err) { 182 | return source, err 183 | } 184 | 185 | return source, fmt.Errorf("unable to get source '%s': %w", namespacedName, err) 186 | } 187 | 188 | source = &bucket 189 | default: 190 | return source, fmt.Errorf("source `%s` kind '%s' not supported", 191 | name, kind) 192 | } 193 | 194 | return source, nil 195 | } 196 | -------------------------------------------------------------------------------- /slack.go: -------------------------------------------------------------------------------- 1 | package variant 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/nlopes/slack" 13 | "github.com/zclconf/go-cty/cty" 14 | "golang.org/x/xerrors" 15 | 16 | "github.com/mumoshu/variant2/pkg/app" 17 | variantslack "github.com/mumoshu/variant2/pkg/slack" 18 | ) 19 | 20 | func (r *Runner) StartSlackbot(name string) error { 21 | // (This is written by calling isatty beforehand and this just overwrite that 22 | r.Interactive = true 23 | 24 | bot := variantslack.New(name, os.Getenv("SLACK_BOT_TOKEN"), os.Getenv("SLACK_VERIFICATION_TOKEN"), func(bot *variantslack.Connection, cmd string, message slack.SlashCommand) string { 25 | var b bytes.Buffer 26 | 27 | err := r.Run(strings.Split(cmd, " "), RunOptions{ 28 | Stdout: &b, 29 | Stderr: &b, 30 | SetOpts: func(opts map[string]cty.Value, pendingOptions []app.PendingInput) error { 31 | var elems []slack.DialogElement 32 | 33 | for _, o := range pendingOptions { 34 | k := o.Name 35 | 36 | var desc string 37 | 38 | if o.Description != nil { 39 | desc = *o.Description 40 | } 41 | 42 | var elem slack.DialogElement 43 | 44 | switch o.Type { 45 | case cty.String, cty.Bool, cty.Number: 46 | elem = slack.DialogInput{ 47 | Label: k, 48 | Placeholder: desc, 49 | Type: slack.InputTypeText, 50 | Name: k, 51 | // Optional: true, 52 | } 53 | default: 54 | elem = slack.DialogInput{ 55 | Label: k, 56 | Placeholder: desc, 57 | Type: slack.InputTypeTextArea, 58 | Name: k, 59 | // Optional: true, 60 | } 61 | } 62 | 63 | elems = append(elems, elem) 64 | } 65 | 66 | callbackID := message.UserID + "_" + message.TriggerID 67 | dialog := slack.Dialog{ 68 | Title: cmd, 69 | SubmitLabel: "Run", 70 | CallbackID: callbackID, 71 | Elements: elems, 72 | NotifyOnCancel: true, 73 | } 74 | 75 | done := make(chan error, 1) 76 | 77 | bot.RegisterInteractionCallbackHandler(callbackID, func(callback slack.InteractionCallback) (interface{}, error) { 78 | if callback.Type == slack.InteractionTypeDialogCancellation { 79 | done <- nil 80 | 81 | return nil, nil 82 | } 83 | 84 | if callback.Type != slack.InteractionTypeDialogSubmission { 85 | return nil, fmt.Errorf("unexpected type of interaction callback: want %s, got %s", slack.InteractionTypeDialogSubmission, callback.Type) 86 | } 87 | 88 | _, transformers, err := app.MakeQuestions(pendingOptions) 89 | if err != nil { 90 | return nil, xerrors.Errorf("making questions: %w", err) 91 | } 92 | 93 | vals := make(map[string]interface{}) 94 | 95 | errs := map[string]error{} 96 | 97 | for _, o := range pendingOptions { 98 | k := o.Name 99 | v := callback.Submission[k] 100 | 101 | if v == "" { 102 | errs[k] = fmt.Errorf("%s is required", k) 103 | 104 | continue 105 | } 106 | 107 | if o.Type == cty.Bool { 108 | b, err := strconv.ParseBool(v) 109 | if err != nil { 110 | errs[k] = err 111 | 112 | continue 113 | } 114 | 115 | vals[k] = b 116 | } else { 117 | vals[k] = v 118 | } 119 | } 120 | 121 | if len(errs) > 0 { 122 | type validateError struct { 123 | Name string `json:"name"` 124 | Error string `json:"error"` 125 | } 126 | 127 | type validateErrorResponse struct { 128 | Errors []validateError `json:"errors"` 129 | } 130 | 131 | var validateErrs []validateError 132 | 133 | for k, e := range errs { 134 | validateErrs = append(validateErrs, validateError{k, e.Error()}) 135 | } 136 | 137 | errResponse := &validateErrorResponse{ 138 | validateErrs, 139 | } 140 | 141 | return errResponse, nil 142 | } 143 | 144 | if err := app.SetOptsFromMap(transformers, opts, vals); err != nil { 145 | return nil, xerrors.Errorf("setting options: %w", err) 146 | } 147 | 148 | done <- nil 149 | 150 | return nil, nil 151 | }) 152 | 153 | if err := bot.Client.OpenDialogContext(context.TODO(), message.TriggerID, dialog); err != nil { 154 | log.Print("open dialog failed: ", err) 155 | 156 | return nil 157 | } 158 | 159 | return <-done 160 | }, 161 | }) 162 | 163 | out := b.String() 164 | 165 | if err != nil { 166 | return err.Error() 167 | } 168 | 169 | return out 170 | }) 171 | 172 | return bot.Run() 173 | } 174 | -------------------------------------------------------------------------------- /test/export/simple/dst/.gitignore: -------------------------------------------------------------------------------- 1 | dst 2 | dst.variant 3 | -------------------------------------------------------------------------------- /test/export/simple/src/src.variant: -------------------------------------------------------------------------------- 1 | option "int1" { 2 | type = number 3 | } 4 | 5 | option "ints1" { 6 | type = list(number) 7 | } 8 | 9 | option "str1" { 10 | type = string 11 | } 12 | 13 | option "strs1" { 14 | type = list(string) 15 | } 16 | 17 | job "test" { 18 | exec { 19 | command = "echo" 20 | args = [ 21 | tostring(opt.int1), 22 | tostring(opt.ints1[0]), 23 | tostring(opt.ints1[1]), 24 | opt.str1, 25 | join("|", opt.strs1) 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /test/shebang/myapp/myapp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env variant 2 | 3 | import = "." 4 | -------------------------------------------------------------------------------- /test/shebang/myapp/myapp.variant: -------------------------------------------------------------------------------- 1 | option "int1" { 2 | type = number 3 | } 4 | 5 | option "ints1" { 6 | type = list(number) 7 | } 8 | 9 | option "str1" { 10 | type = string 11 | } 12 | 13 | option "strs1" { 14 | type = list(string) 15 | } 16 | 17 | job "test" { 18 | exec { 19 | command = "echo" 20 | args = [ 21 | tostring(opt.int1), 22 | tostring(opt.ints1[0]), 23 | tostring(opt.ints1[1]), 24 | opt.str1, 25 | join("|", opt.strs1) 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /test/variant_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "testing" 9 | 10 | "golang.org/x/xerrors" 11 | 12 | variant "github.com/mumoshu/variant2" 13 | ) 14 | 15 | // Building the binary with `go build -o myapp main.go` 16 | // and running 17 | // ./myapp test 18 | // should produce: 19 | // HELLO WORLD 20 | 21 | func TestMustEval(t *testing.T) { 22 | source := ` 23 | job "test" { 24 | exec { 25 | command = "echo" 26 | args = ["HELLO WORLD"] 27 | } 28 | } 29 | ` 30 | 31 | err := variant.MustLoad(variant.FromSource("myapp", source)).Run([]string{"test"}) 32 | if err != nil { 33 | panic(err) 34 | } 35 | 36 | var verr variant.Error 37 | 38 | var code int 39 | 40 | if err != nil { 41 | if ok := errors.As(err, &verr); ok { 42 | code = verr.ExitCode 43 | } else { 44 | code = 1 45 | } 46 | } else { 47 | code = 0 48 | } 49 | 50 | if code != 0 { 51 | t.Errorf("unexpected code: %d", code) 52 | } 53 | } 54 | 55 | func TestNewFile(t *testing.T) { 56 | myapp, err := variant.Load(variant.FromPath("../examples/simple/simple.variant")) 57 | if err != nil { 58 | t.Fatal(err) 59 | } 60 | 61 | stdout := &bytes.Buffer{} 62 | stderr := &bytes.Buffer{} 63 | 64 | if err := myapp.Run([]string{"app", "deploy", "--namespace=default"}, variant.RunOptions{ 65 | Stdout: stdout, 66 | Stderr: stderr, 67 | }); err != nil { 68 | t.Fatal(err) 69 | } 70 | 71 | var verr variant.Error 72 | 73 | var code int 74 | 75 | if err != nil { 76 | if ok := errors.As(err, &verr); ok { 77 | code = verr.ExitCode 78 | } else { 79 | code = 1 80 | } 81 | } else { 82 | code = 0 83 | } 84 | 85 | if code != 0 { 86 | t.Errorf("unexpected code: %d", code) 87 | } 88 | } 89 | 90 | func TestExtensionWithGo(t *testing.T) { 91 | myapp, err := variant.Load(variant.FromPath("../examples/simple/simple.variant")) 92 | if err != nil { 93 | t.Fatal(err) 94 | } 95 | 96 | // outWriter and errWriter are automatically closed by Variant core after the calling anonymous func 97 | // to avoid leaking 98 | myapp.Add( 99 | variant.Job{ 100 | Name: "foo bar", 101 | Description: "foobar", 102 | Options: map[string]variant.Variable{ 103 | "namespace": { 104 | Type: variant.String, 105 | Description: "namespace", 106 | }, 107 | }, 108 | Parameters: map[string]variant.Variable{ 109 | "param1": { 110 | Type: variant.String, 111 | Description: "param1", 112 | }, 113 | }, 114 | Run: func(ctx context.Context, s variant.State) error { 115 | v, ok := s.Options["namespace"] 116 | 117 | if !ok { 118 | return fmt.Errorf("missing option %q", "namespace") 119 | } 120 | 121 | ns := v.(string) 122 | 123 | out, stdoutW := variant.Pipe() 124 | errs, stderrW := variant.Pipe() 125 | 126 | defer s.Stdout.Close() 127 | defer s.Stderr.Close() 128 | 129 | subst := variant.State{ 130 | Parameters: map[string]interface{}{}, 131 | Options: map[string]interface{}{"namespace": ns}, 132 | Stdout: stdoutW, 133 | Stderr: stderrW, 134 | } 135 | j, err := myapp.Job("app deploy", subst) 136 | if err != nil { 137 | return err 138 | } 139 | 140 | if err := j(ctx); err != nil { 141 | return err 142 | } 143 | 144 | o, err := out() 145 | if err != nil { 146 | return err 147 | } 148 | 149 | if _, err := s.Stdout.Write([]byte("OUTPUT: " + o.String())); err != nil { 150 | return xerrors.Errorf("writing stdout: %w", err) 151 | } 152 | 153 | e, err := errs() 154 | if err != nil { 155 | return err 156 | } 157 | 158 | if _, err := s.Stderr.Write([]byte("ERROR: " + e.String())); err != nil { 159 | return xerrors.Errorf("writing stderr: %w", err) 160 | } 161 | 162 | return nil 163 | }, 164 | }) 165 | 166 | getStdout, stdout := variant.Pipe() 167 | getStderr, stderr := variant.Pipe() 168 | 169 | jr, err := myapp.Job("foo bar", variant.State{ 170 | Stdout: stdout, 171 | Stderr: stderr, 172 | Parameters: map[string]interface{}{}, 173 | Options: map[string]interface{}{"namespace": "default"}, 174 | }) 175 | if err != nil { 176 | t.Fatal(err) 177 | } 178 | 179 | if err := jr(context.TODO()); err != nil { 180 | t.Fatal(err) 181 | } 182 | 183 | outs, err := getStdout() 184 | if err != nil { 185 | t.Fatal(err) 186 | } 187 | 188 | outStr := outs.String() 189 | if outStr != "" { 190 | t.Errorf("unexpected stdout: got %q", outStr) 191 | } 192 | 193 | errs, err := getStderr() 194 | if err != nil { 195 | t.Fatal(err) 196 | } 197 | 198 | errStr := errs.String() 199 | if errStr != "" { 200 | t.Errorf("unexpected stderr: got %q", errStr) 201 | } 202 | 203 | var verr variant.Error 204 | 205 | var code int 206 | 207 | if err != nil { 208 | if ok := errors.As(err, &verr); ok { 209 | code = verr.ExitCode 210 | } else { 211 | code = 1 212 | } 213 | } else { 214 | code = 0 215 | } 216 | 217 | if code != 0 { 218 | t.Errorf("unexpected code: %d", code) 219 | } 220 | } 221 | 222 | func TestNewDir(t *testing.T) { 223 | myapp, err := variant.Load(variant.FromPath("../examples/simple")) 224 | if err != nil { 225 | panic(err) 226 | } 227 | 228 | stdout := &bytes.Buffer{} 229 | stderr := &bytes.Buffer{} 230 | 231 | if err := myapp.Run([]string{"app", "deploy", "--namespace=default"}, variant.RunOptions{ 232 | Stdout: stdout, 233 | Stderr: stderr, 234 | }); err != nil { 235 | t.Fatal(err) 236 | } 237 | 238 | var verr variant.Error 239 | 240 | var code int 241 | 242 | if err != nil { 243 | if ok := errors.As(err, &verr); ok { 244 | code = verr.ExitCode 245 | } else { 246 | code = 1 247 | } 248 | } else { 249 | code = 0 250 | } 251 | 252 | if code != 0 { 253 | t.Errorf("unexpected code: %d", code) 254 | } 255 | } 256 | 257 | func TestNewDirCobra(t *testing.T) { 258 | myapp, err := variant.Load(variant.FromPath("../examples/simple")) 259 | if err != nil { 260 | panic(err) 261 | } 262 | 263 | // Returns the *cobra.Command 264 | cmd, err := myapp.Cobra() 265 | if err != nil { 266 | t.Fatal(err) 267 | } 268 | 269 | // You can add any command to the root command with: 270 | // cmd.AddCommand(...) 271 | // See the documentation of cobra for more information. 272 | 273 | cmd.SetArgs([]string{"app", "deploy", "--namespace=default"}) 274 | 275 | if err := cmd.Execute(); err != nil { 276 | t.Fatal(err) 277 | } 278 | } 279 | 280 | // variant.(Must)Eval creates a Variant command from the virtual file name and the source code written in the Variant DSL 281 | // variant.(Must)Load creates a Variant command from a file or a directory 282 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | package variant 2 | 3 | import ( 4 | "context" 5 | "io" 6 | ) 7 | 8 | type State struct { 9 | Stdin io.Reader 10 | Parameters map[string]interface{} 11 | Options map[string]interface{} 12 | Stdout io.WriteCloser 13 | Stderr io.WriteCloser 14 | } 15 | 16 | type Job struct { 17 | Name string 18 | Description string 19 | Options map[string]Variable 20 | Parameters map[string]Variable 21 | Run func(context.Context, State) error 22 | } 23 | 24 | type Type int 25 | 26 | const ( 27 | String Type = iota 28 | Int 29 | StringSlice 30 | IntSlice 31 | Bool 32 | ) 33 | 34 | type Variable struct { 35 | Type Type 36 | Description string 37 | } 38 | 39 | type JobRun func(context.Context) error 40 | --------------------------------------------------------------------------------