├── .gitignore ├── .github ├── RELEASE_TEMPLATE.md └── workflows │ └── main.yaml ├── pkg ├── pipes │ ├── cobra_windows.go │ ├── cobra_unix.go │ ├── exec.go │ ├── karg │ │ └── karg_test.go │ ├── helmvalues_test.go │ ├── file.go │ ├── kubectl.go │ ├── pipes.go │ ├── cobra.go │ └── helmvalues.go ├── filters │ ├── application_test.go │ ├── order_test.go │ ├── patch.go │ ├── order.go │ ├── resourcemeta_test.go │ ├── fieldpath.go │ ├── workload.go │ ├── fieldpath_test.go │ ├── resourcemeta.go │ ├── application.go │ └── filters.go ├── tracing │ └── tracing.go ├── konjure │ ├── writer_test.go │ ├── resource_test.go │ ├── filter.go │ └── resource.go └── api │ └── core │ └── v1beta2 │ ├── groupversion.go │ └── types.go ├── internal ├── application │ ├── types_test.go │ ├── labels.go │ ├── types.go │ └── application.go ├── readers │ ├── kustomize.go │ ├── http.go │ ├── resource.go │ ├── git.go │ ├── helm.go │ ├── readers.go │ ├── options.go │ ├── kubernetes.go │ ├── filter.go │ ├── file.go │ ├── secret.go │ └── jsonnet.go ├── spec │ ├── formatter.go │ └── parser_test.go └── command │ ├── secret.go │ ├── helmvalues.go │ ├── helm.go │ ├── jsonnet.go │ └── root.go ├── goreleaser.yaml ├── main.go ├── go.mod └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /konjure 2 | /dist/ 3 | /.idea/ 4 | -------------------------------------------------------------------------------- /.github/RELEASE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### ✨ Added 2 | 3 | * 4 | 5 | ### 🏗 Changed 6 | 7 | * 8 | 9 | ### ⏳ Deprecated 10 | 11 | * 12 | 13 | ### 🛑 Removed 14 | 15 | * 16 | 17 | ### 🐛 Fixed 18 | 19 | * 20 | 21 | ### 🗝 Security 22 | 23 | * 24 | -------------------------------------------------------------------------------- /pkg/pipes/cobra_windows.go: -------------------------------------------------------------------------------- 1 | package pipes 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "strings" 9 | ) 10 | 11 | func editorCmd(ctx context.Context, filename string) *exec.Cmd { 12 | editor := os.Getenv("EDITOR") 13 | if editor == "" { 14 | editor = "notepad" 15 | } 16 | 17 | var args []string 18 | switch { 19 | case !strings.Contains(editor, " "): 20 | args = append(args, editor, filename) 21 | case !strings.Contains(editor, `"'\`): 22 | args = append(args, strings.Split(editor, " ")...) 23 | args = append(args, filename) 24 | default: 25 | shell := os.Getenv("SHELL") 26 | if shell == "" { 27 | shell = "cmd" 28 | } 29 | args = append(args, shell, "/C", fmt.Sprintf("%s %q", editor, filename)) 30 | } 31 | 32 | return exec.CommandContext(ctx, args[0], args[1:]...) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/pipes/cobra_unix.go: -------------------------------------------------------------------------------- 1 | //go:build darwin || linux 2 | 3 | package pipes 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "os" 9 | "os/exec" 10 | "strings" 11 | ) 12 | 13 | func editorCmd(ctx context.Context, filename string) *exec.Cmd { 14 | editor := os.Getenv("EDITOR") 15 | if editor == "" { 16 | editor = "vi" 17 | } 18 | 19 | var args []string 20 | switch { 21 | case !strings.Contains(editor, " "): 22 | args = append(args, editor, filename) 23 | case !strings.Contains(editor, `"'\`): 24 | args = append(args, strings.Split(editor, " ")...) 25 | args = append(args, filename) 26 | default: 27 | shell := os.Getenv("SHELL") 28 | if shell == "" { 29 | shell = "/bin/bash" 30 | } 31 | args = append(args, shell, "-c", fmt.Sprintf("%s %q", editor, filename)) 32 | } 33 | 34 | return exec.CommandContext(ctx, args[0], args[1:]...) 35 | } 36 | -------------------------------------------------------------------------------- /internal/application/types_test.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGroupKind_String(t *testing.T) { 10 | cases := []struct { 11 | desc string 12 | groupKind GroupKind 13 | expected string 14 | }{ 15 | { 16 | desc: "empty", 17 | expected: ".", 18 | }, 19 | { 20 | // This case is important because of how `kubectl` resolves types: 21 | // for example, `kubectl get Foo` won't work (it's a plain kind, not 22 | // a resource name); but `kubectl get Foo.` will trigger a GVK parse 23 | // that will ultimately resolve to the correct type. 24 | desc: "kind only", 25 | groupKind: GroupKind{Kind: "Foo"}, 26 | expected: "Foo.", 27 | }, 28 | } 29 | for _, tc := range cases { 30 | t.Run(tc.desc, func(t *testing.T) { 31 | assert.Equal(t, tc.expected, tc.groupKind.String()) 32 | }) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | before: 3 | hooks: ['go mod tidy', 'go generate ./...'] 4 | builds: 5 | - env: 6 | - CGO_ENABLED=0 7 | goarch: 8 | - amd64 9 | - arm64 10 | ignore: 11 | - goos: linux 12 | goarch: arm64 13 | archives: 14 | - name_template: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}" 15 | files: 16 | - none* 17 | checksum: 18 | name_template: "checksums.txt" 19 | snapshot: 20 | name_template: "{{ .Tag }}-next+commit.{{ .ShortCommit }}" 21 | release: 22 | draft: true 23 | prerelease: auto 24 | brews: 25 | - repository: 26 | owner: thestormforge 27 | name: homebrew-tap 28 | commit_author: 29 | name: Butch Masters 30 | email: butch@stormforge.io 31 | directory: Formula 32 | homepage: "https://github.com/thestormforge/konjure/" 33 | description: Manifest appear! 34 | install: | 35 | bin.install "konjure" 36 | 37 | # generate and install bash completion 38 | output = Utils.safe_popen_read("#{bin}/konjure", "completion", "bash") 39 | (bash_completion/"konjure").write output 40 | 41 | # generate and install zsh completion 42 | output = Utils.safe_popen_read("#{bin}/konjure", "completion", "zsh") 43 | (zsh_completion/"_konjure").write output 44 | -------------------------------------------------------------------------------- /internal/readers/kustomize.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 21 | "sigs.k8s.io/kustomize/kyaml/yaml" 22 | ) 23 | 24 | type KustomizeReader struct { 25 | konjurev1beta2.Kustomize 26 | Runtime 27 | } 28 | 29 | func (kustomize *KustomizeReader) Read() ([]*yaml.RNode, error) { 30 | cmd := kustomize.command() 31 | cmd.Args = append(cmd.Args, "build", kustomize.Root) 32 | return cmd.Read() 33 | } 34 | 35 | func (kustomize *KustomizeReader) command() *command { 36 | cmd := kustomize.Runtime.command("kustomize") 37 | return cmd 38 | } 39 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "os" 22 | "time" 23 | 24 | "github.com/spf13/cobra" 25 | "github.com/thestormforge/konjure/internal/command" 26 | ) 27 | 28 | var ( 29 | version = "" 30 | commit = "HEAD" 31 | date = time.Now().String() 32 | ) 33 | 34 | func init() { 35 | cobra.EnableCommandSorting = false 36 | } 37 | 38 | func main() { 39 | // TODO Wrap `http.DefaultTransport` so it includes the UA string 40 | 41 | ctx := context.Background() 42 | cmd := command.NewRootCommand(version, commit, date) 43 | if err := cmd.ExecuteContext(ctx); err != nil { 44 | os.Exit(1) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: Main 2 | on: push 3 | jobs: 4 | build: 5 | name: Build 6 | runs-on: ubuntu-latest 7 | steps: 8 | - name: Check out code 9 | uses: actions/checkout@v4 10 | with: 11 | fetch-depth: 0 12 | - name: Set up Go 13 | uses: actions/setup-go@v5 14 | with: 15 | go-version-file: 'go.mod' 16 | - name: Verify 17 | run: | 18 | go mod tidy 19 | go generate ./... 20 | go test ./... 21 | 22 | goreleaserFlags=() 23 | goreleaserFlags+=(--release-notes ./.github/RELEASE_TEMPLATE.md) 24 | if [ "${GITHUB_REF_TYPE}" != "tag" ]; then 25 | goreleaserFlags+=(--snapshot) 26 | fi 27 | echo "GORELEASER_FLAGS=${goreleaserFlags[@]}" >> $GITHUB_ENV 28 | - name: Run GoReleaser 29 | uses: goreleaser/goreleaser-action@v6 30 | with: 31 | version: v2 32 | args: release ${{ env.GORELEASER_FLAGS }} 33 | env: 34 | GITHUB_TOKEN: ${{ secrets.BMASTERS_TOKEN }} 35 | - name: Upload artifacts 36 | uses: actions/upload-artifact@v4 37 | with: 38 | name: dist-archives 39 | path: dist/*.tar.gz 40 | -------------------------------------------------------------------------------- /pkg/filters/application_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestSplitHelmChart(t *testing.T) { 10 | cases := []struct { 11 | desc string 12 | chart string 13 | expectedName string 14 | expectedVersion string 15 | }{ 16 | { 17 | desc: "simple", 18 | chart: "foo-1.0.0", 19 | expectedName: "foo", 20 | expectedVersion: "1.0.0", 21 | }, 22 | { 23 | desc: "prerelease", 24 | chart: "foo-1.0.0-beta.1", 25 | expectedName: "foo", 26 | expectedVersion: "1.0.0-beta.1", 27 | }, 28 | { 29 | desc: "hyphenated name", 30 | chart: "foo-bar-1.0.0", 31 | expectedName: "foo-bar", 32 | expectedVersion: "1.0.0", 33 | }, 34 | { 35 | desc: "no version", 36 | chart: "foo-bar", 37 | expectedName: "foo-bar", 38 | expectedVersion: "", 39 | }, 40 | { 41 | desc: "invalid version", 42 | chart: "foo-bar-01.0.0", 43 | expectedName: "foo-bar-01.0.0", 44 | expectedVersion: "", 45 | }, 46 | } 47 | for _, tc := range cases { 48 | t.Run(tc.desc, func(t *testing.T) { 49 | name, version := splitHelmChart(tc.chart) 50 | assert.Equal(t, tc.expectedName, name, "name") 51 | assert.Equal(t, tc.expectedVersion, version, "version") 52 | }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /internal/application/labels.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | // See: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/ 4 | // See: https://helm.sh/docs/chart_best_practices/labels/ 5 | 6 | const ( 7 | // LabelName is the recommended label for the name of the 8 | // application. For example, `mysql`. 9 | LabelName = "app.kubernetes.io/name" 10 | // LabelInstance is the recommended label for a unique name 11 | // identifying the instance of an application. For example, `mysql-abcxzy`. 12 | LabelInstance = "app.kubernetes.io/instance" 13 | // LabelVersion is the recommended label for the current version 14 | // of the application. For example, `5.7.21`. 15 | LabelVersion = "app.kubernetes.io/version" 16 | // LabelComponent is the recommended label for the component 17 | // within the architecture. For example, `database`. 18 | LabelComponent = "app.kubernetes.io/component" 19 | // LabelPartOf is the recommended label for the name of a higher 20 | // level application this one is part of. For example, `wordpress`. 21 | LabelPartOf = "app.kubernetes.io/part-of" 22 | // LabelManagedBy is the recommended label for the tool being 23 | // used to manage the operation of an application. For example, `helm`. 24 | LabelManagedBy = "app.kubernetes.io/managed-by" 25 | // LabelCreatedBy is the recommended label for the controller/user 26 | // who created this resource. For example, `controller-manager`. 27 | LabelCreatedBy = "app.kubernetes.io/created-by" 28 | // LabelHelmChart is the recommended label for the chart name and version. 29 | // For example, `{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}`. 30 | LabelHelmChart = "helm.sh/chart" 31 | ) 32 | -------------------------------------------------------------------------------- /internal/readers/http.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "fmt" 21 | "net/http" 22 | 23 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 24 | "sigs.k8s.io/kustomize/kyaml/kio" 25 | "sigs.k8s.io/kustomize/kyaml/yaml" 26 | ) 27 | 28 | type HTTPReader struct { 29 | konjurev1beta2.HTTP 30 | Client *http.Client 31 | } 32 | 33 | func (r *HTTPReader) Read() ([]*yaml.RNode, error) { 34 | req, err := http.NewRequest(http.MethodGet, r.HTTP.URL, nil) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | // TODO Set Accept headers for JSON or YAML 40 | 41 | c := r.Client 42 | if c == nil { 43 | c = http.DefaultClient 44 | } 45 | 46 | resp, err := c.Do(req) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | defer resp.Body.Close() 52 | if resp.StatusCode < 200 || resp.StatusCode >= 300 { 53 | return nil, fmt.Errorf("invalid response code for %q: %d", r.HTTP.URL, resp.StatusCode) 54 | } 55 | 56 | // TODO Should we annotate where the (likely non-Konjure) resources originated from? Even if the default writer strips those annotations 57 | return (&kio.ByteReader{Reader: resp.Body}).Read() 58 | } 59 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/thestormforge/konjure 2 | 3 | go 1.24.5 4 | 5 | require ( 6 | github.com/evanphx/json-patch/v5 v5.9.11 7 | github.com/fatih/color v1.18.0 8 | github.com/google/go-jsonnet v0.21.0 9 | github.com/google/uuid v1.6.0 10 | github.com/jsonnet-bundler/jsonnet-bundler v0.6.0 11 | github.com/mattn/go-isatty v0.0.20 12 | github.com/oklog/ulid/v2 v2.1.1 13 | github.com/pkg/errors v0.9.1 14 | github.com/rs/zerolog v1.34.0 15 | github.com/sethvargo/go-password v0.3.1 16 | github.com/spf13/cobra v1.9.1 17 | github.com/stretchr/testify v1.10.0 18 | golang.org/x/sync v0.16.0 19 | k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 20 | sigs.k8s.io/kustomize/kyaml v0.20.1 21 | sigs.k8s.io/yaml v1.6.0 22 | ) 23 | 24 | require ( 25 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 26 | github.com/elliotchance/orderedmap/v2 v2.7.0 // indirect 27 | github.com/go-errors/errors v1.5.1 // indirect 28 | github.com/go-openapi/jsonpointer v0.21.1 // indirect 29 | github.com/go-openapi/jsonreference v0.21.0 // indirect 30 | github.com/go-openapi/swag v0.23.1 // indirect 31 | github.com/google/gnostic-models v0.7.0 // indirect 32 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 33 | github.com/josharian/intern v1.0.0 // indirect 34 | github.com/mailru/easyjson v0.9.0 // indirect 35 | github.com/mattn/go-colorable v0.1.14 // indirect 36 | github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect 37 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 38 | github.com/spf13/pflag v1.0.7 // indirect 39 | github.com/xlab/treeprint v1.2.0 // indirect 40 | go.yaml.in/yaml/v2 v2.4.2 // indirect 41 | go.yaml.in/yaml/v3 v3.0.4 // indirect 42 | golang.org/x/crypto v0.41.0 // indirect 43 | golang.org/x/sys v0.35.0 // indirect 44 | google.golang.org/protobuf v1.36.6 // indirect 45 | gopkg.in/yaml.v3 v3.0.1 // indirect 46 | ) 47 | -------------------------------------------------------------------------------- /internal/spec/formatter.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package spec 18 | 19 | import ( 20 | "fmt" 21 | 22 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 23 | ) 24 | 25 | type Formatter struct { 26 | } 27 | 28 | func (f *Formatter) Encode(obj any) (string, error) { 29 | switch s := obj.(type) { 30 | case *konjurev1beta2.Resource: 31 | if len(s.Resources) == 1 { 32 | return s.Resources[0], nil 33 | } 34 | 35 | case *konjurev1beta2.Helm: 36 | // TODO Should we attempt to make this into a URL? 37 | 38 | case *konjurev1beta2.Jsonnet: 39 | if s.Filename != "" && 40 | s.Code == "" && 41 | len(s.JsonnetPath) == 0 && 42 | len(s.ExternalVariables) == 0 && 43 | len(s.TopLevelArguments) == 0 && 44 | s.JsonnetBundlerPackageHome == "" && 45 | !s.JsonnetBundlerRefresh { 46 | return s.Filename, nil 47 | } 48 | 49 | case *konjurev1beta2.Kubernetes: 50 | // Do nothing 51 | 52 | case *konjurev1beta2.Kustomize: 53 | return s.Root, nil 54 | 55 | case *konjurev1beta2.Secret: 56 | // There is no specification form for secrets 57 | 58 | case *konjurev1beta2.Git: 59 | // TODO This is probably more complex because of all the allowed formats 60 | 61 | case *konjurev1beta2.HTTP: 62 | return s.URL, nil 63 | 64 | case *konjurev1beta2.File: 65 | return s.Path, nil 66 | } 67 | 68 | return "", fmt.Errorf("object cannot be formatted") 69 | } 70 | -------------------------------------------------------------------------------- /pkg/pipes/exec.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pipes 18 | 19 | import ( 20 | "bytes" 21 | "os/exec" 22 | "time" 23 | 24 | "github.com/thestormforge/konjure/pkg/tracing" 25 | "golang.org/x/sync/errgroup" 26 | "sigs.k8s.io/kustomize/kyaml/kio" 27 | "sigs.k8s.io/kustomize/kyaml/yaml" 28 | ) 29 | 30 | // ExecReader is a KYAML reader that consumes YAML from another process via stdout. 31 | type ExecReader struct { 32 | // The YAML producing command. 33 | *exec.Cmd 34 | } 35 | 36 | // Read executes the supplied command and parses the output as a YAML document stream. 37 | func (c *ExecReader) Read() ([]*yaml.RNode, error) { 38 | start := time.Now() 39 | defer tracing.Exec(c.Cmd, start) 40 | data, err := c.Cmd.Output() 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | return (&kio.ByteReader{ 46 | Reader: bytes.NewReader(data), 47 | }).Read() 48 | } 49 | 50 | // ExecWriter is a KYAML writer that sends YAML to another process via stdin. 51 | type ExecWriter struct { 52 | // The YAML consuming command. 53 | *exec.Cmd 54 | } 55 | 56 | // Write executes the supplied command, piping the generated YAML to stdin. 57 | func (c *ExecWriter) Write(nodes []*yaml.RNode) error { 58 | // Open stdin for writing 59 | p, err := c.Cmd.StdinPipe() 60 | if err != nil { 61 | return err 62 | } 63 | 64 | // Start the command 65 | start := time.Now() 66 | if err := c.Cmd.Start(); err != nil { 67 | return err 68 | } 69 | defer tracing.Exec(c.Cmd, start) 70 | 71 | // Sync on the command finishing and/or the byte writer writing 72 | g := errgroup.Group{} 73 | g.Go(c.Cmd.Wait) 74 | g.Go(func() error { 75 | defer p.Close() 76 | return kio.ByteWriter{ 77 | Writer: p, // TODO Use `bufio.NewWriter(p)`? 78 | }.Write(nodes) 79 | }) 80 | return g.Wait() 81 | } 82 | -------------------------------------------------------------------------------- /pkg/pipes/karg/karg_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package karg 18 | 19 | import ( 20 | "os/exec" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | ) 25 | 26 | func TestWithGetOptions(t *testing.T) { 27 | cases := []struct { 28 | desc string 29 | args []string 30 | opts []GetOption 31 | expected []string 32 | }{ 33 | { 34 | desc: "empty", 35 | expected: []string{"kubectl", "get"}, 36 | }, 37 | { 38 | desc: "resource core kind with selector", 39 | opts: []GetOption{ResourceKind("v1", "Namespace"), Selector("foo=bar")}, 40 | expected: []string{"kubectl", "get", "Namespace.v1.", "--selector", "foo=bar"}, 41 | }, 42 | { 43 | desc: "resource apps kind with selector", 44 | opts: []GetOption{ResourceKind("apps/v1", "Deployment"), Selector("foo=bar")}, 45 | expected: []string{"kubectl", "get", "Deployment.v1.apps", "--selector", "foo=bar"}, 46 | }, 47 | { 48 | desc: "resource types with all namespaces", 49 | args: []string{"--namespace", "default"}, 50 | opts: []GetOption{ResourceType("deployments", "statefulsets"), AllNamespaces(true)}, 51 | expected: []string{"kubectl", "get", "deployments,statefulsets", "--all-namespaces"}, 52 | }, 53 | { 54 | desc: "tricky namespace", 55 | args: []string{"--namespace=default"}, 56 | opts: []GetOption{ResourceName("secret", "my-token"), AllNamespaces(true)}, 57 | expected: []string{"kubectl", "get", "secret/my-token", "--all-namespaces"}, 58 | }, 59 | } 60 | for _, tc := range cases { 61 | t.Run(tc.desc, func(t *testing.T) { 62 | cmd := exec.Command("kubectl", append([]string{"get"}, tc.args...)...) 63 | WithGetOptions(cmd, tc.opts...) 64 | assert.Equal(t, tc.expected, cmd.Args) 65 | }) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /internal/application/types.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | 8 | "sigs.k8s.io/kustomize/kyaml/yaml" 9 | ) 10 | 11 | // StripVersion removes the version from a group/version (APIVersion) string. 12 | func StripVersion(gv string) string { 13 | if gv == "" || gv == "v1" { 14 | return "" 15 | } 16 | return strings.Split(gv, "/")[0] 17 | } 18 | 19 | type GroupKind struct { 20 | Group string `yaml:"group"` 21 | Kind string `yaml:"kind"` 22 | } 23 | 24 | // Note that the Application SIG shows the core type case as both "v1" and "core", 25 | // however something like `kubectl get Service.core` or `kubectl get Service.v1` 26 | // will fail while `kubectl get Service.` works. Therefore, empty string. 27 | 28 | func (gk *GroupKind) Matches(t yaml.TypeMeta) bool { 29 | if t.Kind != gk.Kind { 30 | return false 31 | } 32 | 33 | if StripVersion(t.APIVersion) != gk.Group { 34 | return false 35 | } 36 | 37 | return true 38 | } 39 | 40 | func (gk *GroupKind) String() string { 41 | return gk.Kind + "." + gk.Group 42 | } 43 | 44 | type LabelSelector struct { 45 | MatchLabels map[string]string `yaml:"matchLabels"` 46 | MatchExpressions []struct { 47 | Key string `yaml:"key"` 48 | Operator string `yaml:"operator"` 49 | Values []string `yaml:"values"` 50 | } `yaml:"matchExpressions"` 51 | } 52 | 53 | func (ls *LabelSelector) String() string { 54 | if ls == nil || len(ls.MatchLabels)+len(ls.MatchExpressions) == 0 { 55 | return "" 56 | } 57 | var req []string 58 | for k, v := range ls.MatchLabels { 59 | req = append(req, fmt.Sprintf("%s=%s", k, v)) 60 | } 61 | for _, expr := range ls.MatchExpressions { 62 | switch expr.Operator { 63 | case "In": 64 | req = append(req, fmt.Sprintf("%s in (%s)", expr.Key, joinSorted(expr.Values, ","))) 65 | case "NotIn": 66 | req = append(req, fmt.Sprintf("%s notin (%s)", expr.Key, joinSorted(expr.Values, ","))) 67 | case "Exists": 68 | req = append(req, fmt.Sprintf("%s", expr.Key)) 69 | case "DoesNotExist": 70 | req = append(req, fmt.Sprintf("!%s", expr.Key)) 71 | } 72 | } 73 | return strings.Join(req, ",") 74 | } 75 | 76 | func joinSorted(values []string, sep string) string { 77 | if !sort.StringsAreSorted(values) { 78 | sorted := make([]string, len(values)) 79 | copy(sorted, values) 80 | sort.Strings(sorted) 81 | values = sorted 82 | } 83 | return strings.Join(values, sep) 84 | } 85 | -------------------------------------------------------------------------------- /internal/readers/resource.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "io" 21 | "os" 22 | 23 | "github.com/thestormforge/konjure/internal/spec" 24 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 25 | "sigs.k8s.io/kustomize/kyaml/kio" 26 | "sigs.k8s.io/kustomize/kyaml/yaml" 27 | ) 28 | 29 | // ResourceReader generates Konjure resource nodes by parsing the configured resource specifications. 30 | type ResourceReader struct { 31 | // The list of resource specifications to generate Konjure resources from. 32 | konjurev1beta2.Resource 33 | // The byte stream of (possibly non-Konjure) resources to read give the 34 | // resource specification of "-". Defaults to `os.Stdin`. 35 | Reader io.Reader 36 | } 37 | 38 | // Read produces parses resource specifications and returns resource nodes. 39 | func (r *ResourceReader) Read() ([]*yaml.RNode, error) { 40 | result := kio.ResourceNodeSlice{} 41 | 42 | parser := spec.Parser{Reader: r.Reader} 43 | if parser.Reader == nil { 44 | parser.Reader = os.Stdin 45 | } 46 | 47 | for _, res := range r.Resources { 48 | // Parse the resource specification and append the result 49 | res, err := parser.Decode(res) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | switch rn := res.(type) { 55 | 56 | case kio.Reader: 57 | // It is possible the spec parser returns a reader directly (e.g. when reading stdin) 58 | ns, err := rn.Read() 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | result = append(result, ns...) 64 | 65 | default: 66 | // Assume the spec resulted in a Konjure type: GetRNode will fail if it did not 67 | n, err := konjurev1beta2.GetRNode(rn) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | result = append(result, n) 73 | } 74 | } 75 | 76 | return result, nil 77 | } 78 | -------------------------------------------------------------------------------- /internal/readers/git.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "os" 21 | "os/exec" 22 | "path/filepath" 23 | 24 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 25 | "sigs.k8s.io/kustomize/kyaml/yaml" 26 | ) 27 | 28 | type GitReader struct { 29 | konjurev1beta2.Git 30 | path string 31 | } 32 | 33 | func (r *GitReader) Read() ([]*yaml.RNode, error) { 34 | var err error 35 | r.path, err = os.MkdirTemp("", "konjure-git") 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | refspec := r.Refspec 41 | if refspec == "" { 42 | refspec = "HEAD" 43 | } 44 | 45 | // Fetch the into the temporary directory 46 | if err := r.run("init"); err != nil { 47 | return nil, err 48 | } 49 | if err := r.run("remote", "add", "origin", r.Repository); err != nil { 50 | return nil, err 51 | } 52 | if err := r.run("fetch", "--depth=1", "origin", refspec); err != nil { 53 | return nil, err 54 | } 55 | if err := r.run("checkout", "FETCH_HEAD"); err != nil { 56 | return nil, err 57 | } 58 | if err := r.run("submodule", "update", "--init", "--recursive"); err != nil { 59 | return nil, err 60 | } 61 | 62 | // This creates a single File resource for the subdirectory of the Git repository 63 | // TODO Annotate the File resource with the Git information? (that should be whenever a Konjure resource creates another Konjure resource) 64 | n, err := konjurev1beta2.GetRNode(&konjurev1beta2.File{ 65 | Path: filepath.Join(r.path, r.Context), 66 | }) 67 | if err != nil { 68 | return nil, err 69 | } 70 | return []*yaml.RNode{n}, nil 71 | } 72 | 73 | func (r *GitReader) Clean() error { 74 | if r.path == "" { 75 | return nil 76 | } 77 | if err := os.RemoveAll(r.path); err != nil { 78 | return err 79 | } 80 | r.path = "" 81 | return nil 82 | } 83 | 84 | func (r *GitReader) run(arg ...string) error { 85 | cmd := exec.Command("git", arg...) 86 | cmd.Dir = r.path 87 | return cmd.Run() 88 | } 89 | -------------------------------------------------------------------------------- /pkg/tracing/tracing.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package tracing 18 | 19 | import ( 20 | "os/exec" 21 | "syscall" 22 | "time" 23 | 24 | "github.com/rs/zerolog" 25 | ) 26 | 27 | // Log is the debug/trace log used by Konjure. It is initially set to a 28 | // disabled logger but may be globally overridden. 29 | var Log = zerolog.Nop() 30 | 31 | // Exec logs a trace message with request/response fields for the specified 32 | // command (which should already have run when this is called). 33 | func Exec(cmd *exec.Cmd, start time.Time) { 34 | Log.Trace(). 35 | CallerSkipFrame(1). 36 | Func(func(e *zerolog.Event) { 37 | req := zerolog.Dict() 38 | if cmd != nil { 39 | args := zerolog.Arr() 40 | for _, arg := range cmd.Args { 41 | args.Str(arg) 42 | } 43 | req = req. 44 | Str("path", cmd.Path). 45 | Array("args", args). 46 | Str("dir", cmd.Dir) 47 | } 48 | e.Dict("execRequest", req) 49 | }). 50 | Func(func(e *zerolog.Event) { 51 | resp := zerolog.Dict() 52 | if cmd.ProcessState != nil { 53 | resp.Int("pid", cmd.ProcessState.Pid()). 54 | Dur("totalTime", time.Since(start)). 55 | Dur("userTime", cmd.ProcessState.UserTime()). 56 | Dur("systemTime", cmd.ProcessState.SystemTime()) 57 | if ws, ok := cmd.ProcessState.Sys().(syscall.WaitStatus); ok { 58 | switch { 59 | case ws.Exited(): 60 | resp = resp.Int("exitStatus", ws.ExitStatus()) 61 | case ws.Signaled(): 62 | resp = resp.Stringer("signal", ws.Signal()) 63 | case ws.Stopped(): 64 | if ws.StopSignal() == syscall.SIGTRAP && ws.TrapCause() != 0 { 65 | resp = resp.Int("trapCause", ws.TrapCause()) 66 | } else { 67 | resp = resp.Stringer("stopSignal", ws.StopSignal()) 68 | } 69 | case ws.Continued(): 70 | resp = resp.Bool("continued", true) 71 | } 72 | if ws.CoreDump() { 73 | resp = resp.Bool("coreDumped", true) 74 | } 75 | } else { 76 | resp = resp.Int("exitCode", cmd.ProcessState.ExitCode()) 77 | } 78 | } 79 | e.Dict("execResponse", resp) 80 | }). 81 | Msgf("Exit Code: %d", cmd.ProcessState.ExitCode()) 82 | } 83 | -------------------------------------------------------------------------------- /pkg/filters/order_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "sigs.k8s.io/kustomize/kyaml/kio" 8 | "sigs.k8s.io/kustomize/kyaml/yaml" 9 | ) 10 | 11 | func TestSortByKind(t *testing.T) { 12 | cases := []struct { 13 | desc string 14 | sort kio.Filter 15 | resources []yaml.ResourceMeta 16 | expectedNames string 17 | }{ 18 | { 19 | desc: "install order", 20 | sort: InstallOrder(), 21 | resources: []yaml.ResourceMeta{ 22 | kindAndName("APIService", "!"), 23 | kindAndName("Bunny", "!"), 24 | kindAndName("ClusterRole", "l"), 25 | kindAndName("ClusterRoleBinding", "s"), 26 | kindAndName("ClusterRoleBindingList", "t"), 27 | kindAndName("ClusterRoleList", "i"), 28 | kindAndName("ConfigMap", "f"), 29 | kindAndName("CronJob", "o"), 30 | kindAndName("CustomResourceDefinition", "i"), 31 | kindAndName("DaemonSet", "i"), 32 | kindAndName("Deployment", "d"), 33 | kindAndName("Fuzzy", "!"), 34 | kindAndName("HorizontalPodAutoscaler", "o"), 35 | kindAndName("Ingress", "s"), 36 | kindAndName("IngressClass", "u"), 37 | kindAndName("Job", "i"), 38 | kindAndName("LimitRange", "e"), 39 | kindAndName("Namespace", "s"), 40 | kindAndName("NetworkPolicy", "u"), 41 | kindAndName("PersistentVolume", "a"), 42 | kindAndName("PersistentVolumeClaim", "g"), 43 | kindAndName("Pod", "a"), 44 | kindAndName("PodDisruptionBudget", "c"), 45 | kindAndName("PodSecurityPolicy", "r"), 46 | kindAndName("ReplicaSet", "i"), 47 | kindAndName("ReplicationController", "l"), 48 | kindAndName("ResourceQuota", "p"), 49 | kindAndName("Role", "i"), 50 | kindAndName("RoleBinding", "e"), 51 | kindAndName("RoleBindingList", "x"), 52 | kindAndName("RoleList", "c"), 53 | kindAndName("Secret", "l"), 54 | kindAndName("SecretList", "i"), 55 | kindAndName("Service", "p"), 56 | kindAndName("ServiceAccount", "a"), 57 | kindAndName("StatefulSet", "c"), 58 | kindAndName("StorageClass", "r"), 59 | }, 60 | expectedNames: "supercalifragilisticexpialidocious!!!", 61 | }, 62 | } 63 | for _, tc := range cases { 64 | t.Run(tc.desc, func(t *testing.T) { 65 | nodes := make([]*yaml.RNode, 0, len(tc.resources)) 66 | for _, md := range tc.resources { 67 | v := yaml.Node{} 68 | if err := v.Encode(&md); assert.NoError(t, err) { 69 | nodes = append(nodes, yaml.NewRNode(&v)) 70 | } 71 | } 72 | 73 | actualNodes, err := tc.sort.Filter(nodes) 74 | if assert.NoError(t, err) { 75 | actualNames := "" 76 | for _, n := range actualNodes { 77 | actualNames += n.GetName() 78 | } 79 | assert.Equal(t, tc.expectedNames, actualNames) 80 | } 81 | }) 82 | } 83 | } 84 | 85 | func kindAndName(kind, name string) yaml.ResourceMeta { 86 | md := yaml.ResourceMeta{} 87 | md.Kind = kind 88 | md.Name = name 89 | return md 90 | } 91 | -------------------------------------------------------------------------------- /pkg/filters/patch.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | jsonpatch "github.com/evanphx/json-patch/v5" 8 | "sigs.k8s.io/kustomize/kyaml/yaml" 9 | "sigs.k8s.io/kustomize/kyaml/yaml/merge2" 10 | yaml2 "sigs.k8s.io/yaml" 11 | ) 12 | 13 | // UnsupportedPatchError is raised when a patch format is not recognized. 14 | type UnsupportedPatchError struct { 15 | PatchType string 16 | } 17 | 18 | func (e *UnsupportedPatchError) Error() string { 19 | return fmt.Sprintf("unsupported patch type: %q", e.PatchType) 20 | } 21 | 22 | // PatchFilter is used to apply an arbitrary patch. 23 | type PatchFilter struct { 24 | // The media type of the patch being applied. 25 | PatchType string 26 | // The actual raw patch. 27 | PatchData []byte 28 | // Flag that enables strict JSON Patch processing. Default behavior is to allow 29 | // missing remove paths and create missing add paths. 30 | StrictJSONPatch bool 31 | } 32 | 33 | // Filter applies the configured patch. 34 | func (f *PatchFilter) Filter(node *yaml.RNode) (*yaml.RNode, error) { 35 | switch f.PatchType { 36 | case "application/strategic-merge-patch+json", "strategic", "application/merge-patch+json", "merge", "": 37 | // The patch is likely JSON, parse it as YAML and just clear the style 38 | patchNode := yaml.NewRNode(&yaml.Node{}) 39 | if err := yaml.NewDecoder(bytes.NewReader(f.PatchData)).Decode(patchNode.YNode()); err != nil { 40 | return nil, err 41 | } 42 | _ = patchNode.PipeE(ResetStyle()) 43 | 44 | // Strategic Merge/Merge Patch is just the merge2 logic 45 | opts := yaml.MergeOptions{ 46 | ListIncreaseDirection: yaml.MergeOptionsListPrepend, 47 | } 48 | return merge2.Merge(patchNode, node, opts) 49 | 50 | case "application/json-patch+json", "json": 51 | // The patch is likely JSON, but might be YAML that needs to be converted to JSON 52 | patchData := f.PatchData 53 | if !bytes.HasPrefix(patchData, []byte("[")) { 54 | jsonData, err := yaml2.YAMLToJSON(patchData) 55 | if err != nil { 56 | return nil, err 57 | } 58 | patchData = jsonData 59 | } 60 | jsonPatch, err := jsonpatch.DecodePatch(patchData) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | // Adjust strict interpretation of JSON Patch 66 | jsonPatchOptions := jsonpatch.NewApplyOptions() 67 | jsonPatchOptions.AllowMissingPathOnRemove = !f.StrictJSONPatch 68 | jsonPatchOptions.EnsurePathExistsOnAdd = !f.StrictJSONPatch 69 | 70 | // This is going to butcher the YAML ordering/comments/etc. 71 | jsonData, err := node.MarshalJSON() 72 | if err != nil { 73 | return nil, err 74 | } 75 | jsonData, err = jsonPatch.ApplyWithOptions(jsonData, jsonPatchOptions) 76 | if err != nil { 77 | return nil, err 78 | } 79 | err = node.UnmarshalJSON(jsonData) 80 | if err != nil { 81 | return nil, err 82 | } 83 | return node, nil 84 | 85 | default: 86 | // This patch type is not supported 87 | return nil, &UnsupportedPatchError{PatchType: f.PatchType} 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /pkg/filters/order.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "sort" 5 | 6 | "sigs.k8s.io/kustomize/kyaml/kio" 7 | "sigs.k8s.io/kustomize/kyaml/yaml" 8 | ) 9 | 10 | // InstallOrder returns a filter that sorts nodes in the order in which they 11 | // should be created in a cluster. This uses the Helm ordering which is more 12 | // complete than the Kustomize ordering. 13 | func InstallOrder() kio.Filter { 14 | return SortByKind([]string{ 15 | "Namespace", 16 | "NetworkPolicy", 17 | "ResourceQuota", 18 | "LimitRange", 19 | "PodSecurityPolicy", 20 | "PodDisruptionBudget", 21 | "ServiceAccount", 22 | "Secret", 23 | "SecretList", 24 | "ConfigMap", 25 | "StorageClass", 26 | "PersistentVolume", 27 | "PersistentVolumeClaim", 28 | "CustomResourceDefinition", 29 | "ClusterRole", 30 | "ClusterRoleList", 31 | "ClusterRoleBinding", 32 | "ClusterRoleBindingList", 33 | "Role", 34 | "RoleList", 35 | "RoleBinding", 36 | "RoleBindingList", 37 | "Service", 38 | "DaemonSet", 39 | "Pod", 40 | "ReplicationController", 41 | "ReplicaSet", 42 | "Deployment", 43 | "HorizontalPodAutoscaler", 44 | "StatefulSet", 45 | "Job", 46 | "CronJob", 47 | "IngressClass", 48 | "Ingress", 49 | "APIService", 50 | }) 51 | } 52 | 53 | // UninstallOrder returns a filter that sorts nodes in the order in which they 54 | // should be deleted from a cluster. This is not directly the reverse of the 55 | // installation order. 56 | func UninstallOrder() kio.Filter { 57 | return SortByKind([]string{ 58 | "APIService", 59 | "Ingress", 60 | "IngressClass", 61 | "Service", 62 | "CronJob", 63 | "Job", 64 | "StatefulSet", 65 | "HorizontalPodAutoscaler", 66 | "Deployment", 67 | "ReplicaSet", 68 | "ReplicationController", 69 | "Pod", 70 | "DaemonSet", 71 | "RoleBindingList", 72 | "RoleBinding", 73 | "RoleList", 74 | "Role", 75 | "ClusterRoleBindingList", 76 | "ClusterRoleBinding", 77 | "ClusterRoleList", 78 | "ClusterRole", 79 | "CustomResourceDefinition", 80 | "PersistentVolumeClaim", 81 | "PersistentVolume", 82 | "StorageClass", 83 | "ConfigMap", 84 | "SecretList", 85 | "Secret", 86 | "ServiceAccount", 87 | "PodDisruptionBudget", 88 | "PodSecurityPolicy", 89 | "LimitRange", 90 | "ResourceQuota", 91 | "NetworkPolicy", 92 | "Namespace", 93 | }) 94 | } 95 | 96 | // SortByKind returns a filter that sorts nodes based on the supplied list of kinds. 97 | func SortByKind(priority []string) kio.Filter { 98 | order := make(map[string]int, len(priority)) 99 | for i, p := range priority { 100 | order[p] = len(priority) - i 101 | } 102 | 103 | return kio.FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 104 | sort.SliceStable(nodes, func(i, j int) bool { 105 | ki, kj := nodes[i].GetKind(), nodes[j].GetKind() 106 | oi, oj := order[ki], order[kj] 107 | if oi == oj { 108 | return ki < kj 109 | } 110 | return oi >= oj 111 | }) 112 | return nodes, nil 113 | }) 114 | } 115 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🧙‍ Konjure 2 | 3 | ![](https://github.com/thestormforge/konjure/workflows/Main/badge.svg) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/thestormforge/konjure)](https://goreportcard.com/report/github.com/thestormforge/konjure) 5 | 6 | Konjure generates and transforms Kubernetes resource definitions. It can be used as a standalone utility or can be integrated into your GitOps workflows. 7 | 8 | ## Installation 9 | 10 | ### Binaries 11 | 12 | Each [release](https://github.com/thestormforge/konjure/releases/) includes binaries for multiple OS/Arch combinations which can be manually downloaded and installed. 13 | 14 | 1. Download the appropriate binary for your platform from the [releases page](https://github.com/thestormforge/konjure/releases/). 15 | 2. Unpack it, for example: `tar -xzf konjure-linux-amd64.tar.gz` 16 | 3. Move the `konjure` binary to the desired location, for example: `/usr/local/bin/konjure` 17 | 18 | ### Homebrew (macOS) 19 | 20 | Install via the StormForge Tap: 21 | 22 | ```shell 23 | brew install thestormforge/tap/konjure 24 | ``` 25 | 26 | ## Usage 27 | 28 | Konjure can be used to aggregate or generate Kubernetes manifests from a number of different sources. Simply invoke Konjure with a list of sources. 29 | 30 | In the simplest form, Konjure acts like `cat` for Kubernetes manifests, for example the following will emit a YAML document stream with the resources from two files: 31 | 32 | ```shell 33 | konjure service.yaml deployment.yaml 34 | ``` 35 | 36 | Konjure can convert the resources into [NDJSON](http://ndjson.org/) (Newline Delimited JSON) using the `--output ndjson` option (for example, to pipe into [`jq -s`](https://stedolan.github.io/jq/)). It can also apply some basic filters such as `--format` (for consistent field ordering and YAML formatting conventions) or `--keep-comments=false` (to strip comments); use `konjure --help` to see additional options. 37 | 38 | ### Konjure Sources 39 | 40 | In addition to the local file system, Konjure supports pulling resources from the following sources: 41 | 42 | * Local directories 43 | * Git repositories 44 | * HTTP resources 45 | * Helm charts (via `helm template`) 46 | * Kustomize 47 | * Kubernetes 48 | * Jsonnet 49 | 50 | Konjure also has its own resource generators: 51 | 52 | * Secret generator 53 | 54 | Some sources can be specified using a URL: file system paths, HTTP URLs, and Git repository URLs can all be entered directly. Helm chart URLs can also be used when prefixed with `helm::`. 55 | 56 | ### Konjure Resources 57 | 58 | Konjure defines several Kubernetes-like resources which will be expanded in place during execution. For example, if Konjure encounters a resource with the `apiVersion: konjure.stormforge.io/v1beta2` and the `kind: File` it will be replaced with the manifests found in the named file. Konjure resources are expanded iteratively, by using the `--depth N` option you can limit the number of expansions (for example, `--depth 0` is useful for creating a Konjure resource equivalent to the current invocation of Konjure). 59 | 60 | The current (and evolving) definitions can be found in the [API source](pkg/api/core/v1beta2/types.go). 61 | -------------------------------------------------------------------------------- /internal/command/secret.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package command 18 | 19 | import ( 20 | "fmt" 21 | "strconv" 22 | "strings" 23 | 24 | "github.com/spf13/cobra" 25 | "github.com/thestormforge/konjure/internal/readers" 26 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 27 | "github.com/thestormforge/konjure/pkg/konjure" 28 | "sigs.k8s.io/kustomize/kyaml/kio" 29 | ) 30 | 31 | func NewSecretCommand() *cobra.Command { 32 | f := secretFlags{} 33 | 34 | cmd := &cobra.Command{ 35 | Use: "secret", 36 | Short: "Generate secrets", 37 | PreRun: f.preRun, 38 | RunE: func(cmd *cobra.Command, args []string) error { 39 | return kio.Pipeline{ 40 | Inputs: []kio.Reader{&readers.SecretReader{Secret: f.Secret}}, 41 | Outputs: []kio.Writer{&konjure.Writer{Writer: cmd.OutOrStdout()}}, 42 | }.Execute() 43 | }, 44 | } 45 | 46 | cmd.Flags().StringVar(&f.SecretName, "name", "", "`name` of the secret to generate") 47 | cmd.Flags().StringToStringVar(&f.literals, "literal", nil, "literal `name=value` pair") 48 | cmd.Flags().StringArrayVar(&f.FileSources, "file", nil, "file `path` to include") 49 | cmd.Flags().StringArrayVar(&f.EnvSources, "env", nil, "env `file` to read") 50 | cmd.Flags().StringArrayVar(&f.UUIDSources, "uuid", nil, "UUID `key` to generate") 51 | cmd.Flags().StringArrayVar(&f.ULIDSources, "ulid", nil, "ULID `key` to generate") 52 | cmd.Flags().StringToStringVar(&f.passwords, "password", nil, "password `spec` to generate, e.g. 'mypassword=length:5,numDigits:2'") 53 | 54 | return cmd 55 | } 56 | 57 | type secretFlags struct { 58 | konjurev1beta2.Secret 59 | literals map[string]string 60 | passwords map[string]string 61 | } 62 | 63 | func (f *secretFlags) preRun(*cobra.Command, []string) { 64 | for k, v := range f.literals { 65 | f.LiteralSources = append(f.LiteralSources, fmt.Sprintf("%s=%s", k, v)) 66 | } 67 | 68 | for k, v := range f.passwords { 69 | r := konjurev1beta2.PasswordRecipe{Key: k} 70 | for _, s := range strings.Split(v, ",") { 71 | p := strings.SplitN(s, ":", 2) 72 | if len(p) != 2 { 73 | continue 74 | } 75 | 76 | switch p[0] { 77 | case "length": 78 | l, _ := strconv.Atoi(p[1]) 79 | r.Length = &l 80 | case "numDigits": 81 | nd, _ := strconv.Atoi(p[1]) 82 | r.NumDigits = &nd 83 | case "numSymbols": 84 | ns, _ := strconv.Atoi(p[1]) 85 | r.NumSymbols = &ns 86 | case "noUpper": 87 | nu, _ := strconv.ParseBool(p[1]) 88 | r.NoUpper = &nu 89 | case "allowRepeat": 90 | ar, _ := strconv.ParseBool(p[1]) 91 | r.AllowRepeat = &ar 92 | } 93 | } 94 | 95 | f.PasswordSources = append(f.PasswordSources, r) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /internal/readers/helm.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "fmt" 21 | "path/filepath" 22 | 23 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 24 | "github.com/thestormforge/konjure/pkg/filters" 25 | "sigs.k8s.io/kustomize/kyaml/kio" 26 | "sigs.k8s.io/kustomize/kyaml/yaml" 27 | ) 28 | 29 | type HelmReader struct { 30 | konjurev1beta2.Helm 31 | Runtime 32 | 33 | // The path to the Helm repository cache. Corresponds to the `helm --repository-cache` option. 34 | RepositoryCache string 35 | } 36 | 37 | func (helm *HelmReader) Read() ([]*yaml.RNode, error) { 38 | cmd := helm.command() 39 | 40 | cmd.Args = append(cmd.Args, "template") 41 | 42 | if helm.ReleaseName != "" { 43 | cmd.Args = append(cmd.Args, helm.ReleaseName) 44 | } else { 45 | cmd.Args = append(cmd.Args, "--generate-name") 46 | } 47 | 48 | cmd.Args = append(cmd.Args, helm.Chart) 49 | 50 | if helm.Version != "" { 51 | cmd.Args = append(cmd.Args, "--version", helm.Version) 52 | } 53 | 54 | if helm.ReleaseNamespace != "" { 55 | cmd.Args = append(cmd.Args, "--namespace", helm.ReleaseNamespace) 56 | } 57 | 58 | if helm.Repository != "" { 59 | cmd.Args = append(cmd.Args, "--repo", helm.Repository) 60 | } 61 | 62 | for i := range helm.Values { 63 | switch { 64 | 65 | case helm.Values[i].File != "": 66 | // Try to expand a glob; if it fails or does not match, pass on the raw value and let Helm figure it out 67 | valueFiles := []string{helm.Values[i].File} 68 | if matches, err := filepath.Glob(helm.Values[i].File); err == nil && len(matches) > 0 { 69 | valueFiles = matches 70 | } 71 | for _, f := range valueFiles { 72 | cmd.Args = append(cmd.Args, "--values", f) 73 | } 74 | 75 | case helm.Values[i].Name != "": 76 | setOpt := "--set" 77 | if helm.Values[i].LoadFile { 78 | setOpt = "--set-file" 79 | } else if helm.Values[i].ForceString { 80 | setOpt = "--set-string" 81 | } 82 | 83 | cmd.Args = append(cmd.Args, setOpt, fmt.Sprintf("%s=%v", helm.Values[i].Name, helm.Values[i].Value)) 84 | 85 | } 86 | } 87 | 88 | p := &filters.Pipeline{Inputs: []kio.Reader{cmd}} 89 | 90 | if !helm.IncludeTests { 91 | p.Filters = append(p.Filters, &filters.ResourceMetaFilter{ 92 | AnnotationSelector: "helm.sh/hook notin (test-success, test-failure)", 93 | }) 94 | } 95 | 96 | return p.Read() 97 | } 98 | 99 | func (helm *HelmReader) command() *command { 100 | cmd := helm.Runtime.command("helm") 101 | if helm.RepositoryCache != "" { 102 | cmd.Env = append(cmd.Env, "HELM_REPOSITORY_CACHE="+helm.RepositoryCache) 103 | } 104 | return cmd 105 | } 106 | -------------------------------------------------------------------------------- /internal/command/helmvalues.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package command 18 | 19 | import ( 20 | "io" 21 | "os" 22 | 23 | "github.com/spf13/cobra" 24 | "github.com/thestormforge/konjure/pkg/filters" 25 | "github.com/thestormforge/konjure/pkg/konjure" 26 | "github.com/thestormforge/konjure/pkg/pipes" 27 | "k8s.io/kube-openapi/pkg/validation/spec" 28 | "sigs.k8s.io/kustomize/kyaml/kio" 29 | ) 30 | 31 | func NewHelmValuesCommand() *cobra.Command { 32 | var ( 33 | valueOptions pipes.HelmValues 34 | schema string 35 | w konjure.GroupWriter 36 | inPlace bool 37 | ) 38 | 39 | cmd := &cobra.Command{ 40 | Use: "helm-values FILE ...", 41 | Short: "Merge Helm values.yaml files", 42 | Aliases: []string{"values"}, 43 | } 44 | 45 | cmd.Flags().StringSliceVarP(&valueOptions.ValueFiles, "values", "f", []string{}, "specify values in a YAML `file` (can specify multiple)") 46 | cmd.Flags().StringArrayVar(&valueOptions.Values, "set", []string{}, "set values on the command line (for example, `key1=val1`,key2=val2,...)") 47 | cmd.Flags().StringArrayVar(&valueOptions.StringValues, "set-string", []string{}, "set STRING values on the command line (for example, `key1=val1`,key2=val2,...)") 48 | cmd.Flags().StringArrayVar(&valueOptions.FileValues, "set-file", []string{}, "set values from respective files specified via the command line (for example, `key1=path1`,key2=path2,...)") 49 | cmd.Flags().StringVar(&schema, "schema", "", "the values.schema.json `file`; only necessary if it includes Kubernetes extensions with merge instructions") 50 | cmd.Flags().BoolVarP(&inPlace, "in-place", "i", false, "edit files in-place (if multiple FILE arguments are supplied, only the last file is overwritten)") 51 | 52 | cmd.RunE = func(cmd *cobra.Command, args []string) error { 53 | // Configure the writer to overwrite 54 | w.GroupWriter = func(name string) (io.Writer, error) { 55 | if inPlace { 56 | return os.Create(name) 57 | } 58 | return &nopCloser{Writer: cmd.OutOrStdout()}, nil 59 | } 60 | 61 | // Load a subset of the values.schema.json file for merging (if provided) 62 | var s *spec.Schema 63 | if schema != "" { 64 | data, err := os.ReadFile(schema) 65 | if err != nil { 66 | return err 67 | } 68 | s = &spec.Schema{} 69 | if err := s.UnmarshalJSON(data); err != nil { 70 | return err 71 | } 72 | } 73 | 74 | // The file _arguments_ are merged first (i.e. including comments), while `--values` files are just merged together 75 | return kio.Pipeline{ 76 | Inputs: append(pipes.CommandReaders(cmd, args), &valueOptions), 77 | Filters: []kio.Filter{filters.Flatten(s)}, 78 | Outputs: []kio.Writer{&w}, 79 | }.Execute() 80 | } 81 | return cmd 82 | } 83 | 84 | // nopCloser is used to block callers from closing stdout. 85 | type nopCloser struct{ io.Writer } 86 | 87 | // Close does nothing. 88 | func (nopCloser) Close() error { return nil } 89 | -------------------------------------------------------------------------------- /pkg/pipes/helmvalues_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pipes 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | "github.com/stretchr/testify/require" 24 | "github.com/thestormforge/konjure/pkg/filters" 25 | "sigs.k8s.io/kustomize/kyaml/kio" 26 | "sigs.k8s.io/kustomize/kyaml/yaml" 27 | ) 28 | 29 | func TestHelmValues_Read(t *testing.T) { 30 | cases := []struct { 31 | desc string 32 | reader HelmValues 33 | expected []*yaml.RNode 34 | }{ 35 | { 36 | desc: "single flat set", 37 | reader: HelmValues{ 38 | Values: []string{ 39 | "foo=bar", 40 | }, 41 | }, 42 | expected: []*yaml.RNode{yaml.MustParse(`foo: bar`)}, 43 | }, 44 | { 45 | desc: "multiple flat set", 46 | reader: HelmValues{ 47 | Values: []string{ 48 | "a=b", 49 | "c=d", 50 | }, 51 | }, 52 | expected: []*yaml.RNode{ 53 | yaml.MustParse(` 54 | a: b 55 | c: d`), 56 | }, 57 | }, 58 | { 59 | desc: "single nested set", 60 | reader: HelmValues{ 61 | Values: []string{ 62 | "foo[0]=bar", 63 | }, 64 | }, 65 | expected: []*yaml.RNode{yaml.MustParse(`foo: 66 | - bar`)}, 67 | }, 68 | } 69 | for _, tc := range cases { 70 | t.Run(tc.desc, func(t *testing.T) { 71 | actual, err := tc.reader.Read() 72 | if assert.NoError(t, err) { 73 | actualString, err := kio.StringAll(actual) 74 | require.NoError(t, err, "failed to string node") 75 | expectedString, err := kio.StringAll(tc.expected) 76 | require.NoError(t, err, "failed to string node") 77 | assert.YAMLEq(t, expectedString, actualString) 78 | } 79 | }) 80 | } 81 | } 82 | 83 | func TestHelmValues_Apply(t *testing.T) { 84 | nodes, err := (&filters.Pipeline{ 85 | Inputs: []kio.Reader{ 86 | &HelmValues{Values: []string{"a=b"}}, 87 | }, 88 | Filters: []kio.Filter{ 89 | kio.FilterAll((&HelmValues{Values: []string{ 90 | "c.d=e", 91 | "a=z", 92 | }}).Apply()), 93 | }, 94 | }).Read() 95 | assert.NoError(t, err, "failed to apply") 96 | 97 | actual, err := nodes[0].MarshalJSON() 98 | assert.NoError(t, err, "failed to produce JSON") 99 | assert.JSONEq(t, `{"a":"z","c":{"d":"e"}}`, string(actual)) 100 | } 101 | 102 | func TestHelmValues_Flatten(t *testing.T) { 103 | flattened, err := (&filters.Pipeline{ 104 | Inputs: []kio.Reader{ 105 | &HelmValues{Values: []string{"a=b"}}, 106 | &HelmValues{Values: []string{"c=d"}}, 107 | }, 108 | Filters: []kio.Filter{(&HelmValues{}).Flatten()}, 109 | }).Read() 110 | require.NoError(t, err, "failed to flatten output") 111 | assert.Len(t, flattened, 1) 112 | 113 | out := struct { 114 | A string `yaml:"a"` 115 | C string `yaml:"c"` 116 | }{} 117 | err = flattened[0].YNode().Decode(&out) 118 | require.NoError(t, err, "failed to decode result") 119 | assert.Equal(t, "b", out.A) 120 | assert.Equal(t, "d", out.C) 121 | } 122 | -------------------------------------------------------------------------------- /pkg/filters/resourcemeta_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package filters 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | "sigs.k8s.io/kustomize/kyaml/yaml" 24 | ) 25 | 26 | func TestResourceMetaFilter_Filter(t *testing.T) { 27 | cases := []struct { 28 | desc string 29 | filter ResourceMetaFilter 30 | input []*yaml.RNode 31 | expected []*yaml.RNode 32 | }{ 33 | { 34 | desc: "match all", 35 | input: []*yaml.RNode{ 36 | rmNode("test", nil, nil), 37 | }, 38 | expected: []*yaml.RNode{ 39 | rmNode("test", nil, nil), 40 | }, 41 | }, 42 | { 43 | desc: "match name", 44 | filter: ResourceMetaFilter{ 45 | Name: "foo.*", 46 | }, 47 | input: []*yaml.RNode{ 48 | rmNode("foobar", nil, nil), 49 | rmNode("barfoo", nil, nil), 50 | }, 51 | expected: []*yaml.RNode{ 52 | rmNode("foobar", nil, nil), 53 | }, 54 | }, 55 | { 56 | desc: "match name negate", 57 | filter: ResourceMetaFilter{ 58 | Name: "foo.*", 59 | InvertMatch: true, 60 | }, 61 | input: []*yaml.RNode{ 62 | rmNode("foobar", nil, nil), 63 | rmNode("barfoo", nil, nil), 64 | }, 65 | expected: []*yaml.RNode{ 66 | rmNode("barfoo", nil, nil), 67 | }, 68 | }, 69 | { 70 | desc: "match annotation", 71 | filter: ResourceMetaFilter{ 72 | AnnotationSelector: "test=testing", 73 | }, 74 | input: []*yaml.RNode{ 75 | rmNode("test", nil, nil), 76 | rmNode("testWithAnnotation", nil, map[string]string{"test": "testing"}), 77 | }, 78 | expected: []*yaml.RNode{ 79 | rmNode("testWithAnnotation", nil, map[string]string{"test": "testing"}), 80 | }, 81 | }, 82 | { 83 | desc: "match annotation negate", 84 | filter: ResourceMetaFilter{ 85 | AnnotationSelector: "test!=testing", 86 | }, 87 | input: []*yaml.RNode{ 88 | rmNode("test", nil, nil), 89 | rmNode("testWithAnnotation", nil, map[string]string{"test": "testing"}), 90 | }, 91 | expected: []*yaml.RNode{ 92 | rmNode("test", nil, nil), 93 | }, 94 | }, 95 | } 96 | for _, c := range cases { 97 | t.Run(c.desc, func(t *testing.T) { 98 | actual, err := c.filter.Filter(c.input) 99 | if assert.NoError(t, err) { 100 | assert.Equal(t, c.expected, actual) 101 | } 102 | }) 103 | } 104 | } 105 | 106 | // node returns an RNode representing the supplied resource metadata. 107 | func rmNode(name string, labels, annotations map[string]string) *yaml.RNode { 108 | data, err := yaml.Marshal(&yaml.ResourceMeta{ 109 | TypeMeta: yaml.TypeMeta{APIVersion: "invalid.example.com/v1", Kind: "Test"}, 110 | ObjectMeta: yaml.ObjectMeta{ 111 | NameMeta: yaml.NameMeta{ 112 | Name: name, 113 | }, 114 | Labels: labels, 115 | Annotations: annotations, 116 | }, 117 | }) 118 | if err != nil { 119 | panic(err) 120 | } 121 | return yaml.MustParse(string(data)) 122 | } 123 | -------------------------------------------------------------------------------- /pkg/konjure/writer_test.go: -------------------------------------------------------------------------------- 1 | package konjure 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | "sigs.k8s.io/kustomize/kyaml/kio" 9 | "sigs.k8s.io/kustomize/kyaml/yaml" 10 | ) 11 | 12 | func TestRestoreWhiteSpace(t *testing.T) { 13 | cases := []struct { 14 | desc string 15 | roundTrippable bool 16 | input string 17 | }{ 18 | { 19 | desc: "multiple blank lines", 20 | roundTrippable: true, 21 | input: `test: 22 | - foo 23 | 24 | 25 | 26 | test2: 27 | - bar 28 | `, 29 | }, 30 | { 31 | desc: "single line head comment", 32 | roundTrippable: true, 33 | input: `a: b 34 | 35 | # foobar 36 | c: d 37 | `, 38 | }, 39 | { 40 | desc: "multi line head comment", 41 | roundTrippable: true, 42 | input: `a: b 43 | 44 | # foo 45 | # bar 46 | c: d 47 | `, 48 | }, 49 | { 50 | desc: "multi line head comment and multiple blank lines", 51 | roundTrippable: true, 52 | input: `a: b 53 | 54 | 55 | # foo 56 | # bar 57 | c: d 58 | `, 59 | }, 60 | { 61 | desc: "single line foot comment", 62 | roundTrippable: true, 63 | input: `a: b 64 | # foo 65 | 66 | c: d 67 | `, 68 | }, 69 | { 70 | desc: "multi line foot comment", 71 | roundTrippable: true, 72 | input: `a: b 73 | # foo 74 | # bar 75 | 76 | c: d 77 | `, 78 | }, 79 | { 80 | desc: "multi line foot comment and multiple blank lines", 81 | roundTrippable: true, 82 | input: `a: b 83 | # foo 84 | # bar 85 | 86 | 87 | c: d 88 | `, 89 | }, 90 | } 91 | 92 | for _, c := range cases { 93 | t.Run(c.desc, func(t *testing.T) { 94 | node, err := yaml.Parse(c.input) 95 | require.NoError(t, err, "invalid test input YAML") 96 | restoreVerticalWhiteSpace([]*yaml.RNode{node}) 97 | actual, err := kio.StringAll([]*yaml.RNode{node}) 98 | require.NoError(t, err, "failed to format YAML") 99 | if c.roundTrippable { 100 | assert.Equal(t, c.input, actual) 101 | } else { 102 | assert.NotEqual(t, c.input, actual) 103 | } 104 | }) 105 | } 106 | } 107 | 108 | func TestSplitColumns(t *testing.T) { 109 | cases := []struct { 110 | desc string 111 | spec string 112 | expectedHeaders []string 113 | expectedColumns []string 114 | }{ 115 | { 116 | desc: "default headers", 117 | spec: "foo, bar", 118 | expectedHeaders: []string{"FOO", "BAR"}, 119 | expectedColumns: []string{"foo", "bar"}, 120 | }, 121 | { 122 | desc: "default headers from path", 123 | spec: "a.b.c.foo, x.y.z.bar", 124 | expectedHeaders: []string{"FOO", "BAR"}, 125 | expectedColumns: []string{"a.b.c.foo", "x.y.z.bar"}, 126 | }, 127 | { 128 | desc: "explicit headers", 129 | spec: "Foo:a.b.c.foo , Bar:x.y.z.bar", 130 | expectedHeaders: []string{"Foo", "Bar"}, 131 | expectedColumns: []string{"a.b.c.foo", "x.y.z.bar"}, 132 | }, 133 | { 134 | desc: "escaped column", 135 | spec: ":x:y:z, :a:b.c", 136 | expectedHeaders: []string{"X:Y:Z", "C"}, 137 | expectedColumns: []string{"x:y:z", "a:b.c"}, 138 | }, 139 | } 140 | for _, tc := range cases { 141 | t.Run(tc.desc, func(t *testing.T) { 142 | headers, columns := splitColumns(tc.spec) 143 | assert.Equal(t, tc.expectedHeaders, headers) 144 | assert.Equal(t, tc.expectedColumns, columns) 145 | }) 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /internal/command/helm.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package command 18 | 19 | import ( 20 | "github.com/spf13/cobra" 21 | "github.com/thestormforge/konjure/internal/readers" 22 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 23 | "github.com/thestormforge/konjure/pkg/konjure" 24 | "sigs.k8s.io/kustomize/kyaml/kio" 25 | ) 26 | 27 | func NewHelmCommand() *cobra.Command { 28 | f := helmFlags{} 29 | 30 | cmd := &cobra.Command{ 31 | Use: "helm CHART", 32 | Short: "Inflate a Helm chart", 33 | Args: cobra.ExactArgs(1), 34 | PreRun: f.preRun, 35 | RunE: func(cmd *cobra.Command, args []string) error { 36 | return kio.Pipeline{ 37 | Inputs: []kio.Reader{&f.HelmReader}, 38 | Outputs: []kio.Writer{&konjure.Writer{Writer: cmd.OutOrStdout()}}, 39 | }.Execute() 40 | }, 41 | } 42 | 43 | // These flags match what real Helm has 44 | cmd.Flags().StringVar(&f.Helm.Repository, "repo", "", "repository `url` used to locate the chart") 45 | cmd.Flags().StringVar(&f.Helm.ReleaseName, "name", "RELEASE-NAME", "release `name`") 46 | cmd.Flags().StringVarP(&f.Helm.ReleaseNamespace, "namespace", "n", "default", "release `namespace`") 47 | cmd.Flags().StringVar(&f.Helm.Version, "version", "", "fetch a specific `version` of a chart; if empty, the latest version of the chart will be used") 48 | cmd.Flags().StringVar(&f.RepositoryCache, "repository-cache", "", "override the `directory` of your cached Helm repository index") 49 | cmd.Flags().StringToStringVar(&f.set, "set", nil, "set `value`s on the command line") 50 | cmd.Flags().StringToStringVar(&f.setFile, "set-file", nil, "set values from `file`s on the command line") 51 | cmd.Flags().StringToStringVar(&f.setString, "set-string", nil, "set string `value`s on the command line") 52 | cmd.Flags().StringArrayVarP(&f.values, "values", "f", nil, "specify values in a YAML `file`") 53 | 54 | // These flags are specific to our plugin 55 | cmd.Flags().BoolVar(&f.Helm.IncludeTests, "include-tests", false, "do not remove resources labeled as test hooks") 56 | 57 | return cmd 58 | } 59 | 60 | // helmFlags is an extra structure for storing command line options. Unlike real Helm, we don't preserve order of set flags! 61 | type helmFlags struct { 62 | readers.HelmReader 63 | set map[string]string 64 | setFile map[string]string 65 | setString map[string]string 66 | values []string 67 | } 68 | 69 | func (f *helmFlags) preRun(_ *cobra.Command, args []string) { 70 | if len(args) > 0 { 71 | f.Chart = args[0] 72 | } 73 | 74 | for k, v := range f.set { 75 | f.Values = append(f.Values, konjurev1beta2.HelmValue{Name: k, Value: v}) 76 | } 77 | 78 | for k, v := range f.setFile { 79 | f.Values = append(f.Values, konjurev1beta2.HelmValue{Name: k, Value: v, LoadFile: true}) 80 | } 81 | 82 | for k, v := range f.setString { 83 | f.Values = append(f.Values, konjurev1beta2.HelmValue{Name: k, Value: v, ForceString: true}) 84 | } 85 | 86 | for _, valueFile := range f.values { 87 | f.Values = append(f.Values, konjurev1beta2.HelmValue{File: valueFile}) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /internal/application/application.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | import ( 4 | _ "embed" 5 | 6 | "sigs.k8s.io/kustomize/kyaml/openapi" 7 | "sigs.k8s.io/kustomize/kyaml/yaml" 8 | "sigs.k8s.io/kustomize/kyaml/yaml/merge2" 9 | ) 10 | 11 | //go:embed application_schema.json 12 | var schema []byte 13 | 14 | func init() { 15 | // Add our own schema that includes the merge directives necessary for getting 16 | // correct behavior out of the merges. 17 | if err := openapi.AddSchema(schema); err != nil { 18 | panic(err) 19 | } 20 | } 21 | 22 | // Index moves the application resources from a collection of nodes 23 | // and into an indexed map. 24 | func Index(nodes []*yaml.RNode, apps map[yaml.NameMeta]*Node) ([]*yaml.RNode, error) { 25 | var i int 26 | for _, node := range nodes { 27 | md, err := node.GetMeta() 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | // Leave non-Application nodes alone 33 | if md.APIVersion != "app.k8s.io/v1beta1" || md.Kind != "Application" { 34 | nodes[i] = node 35 | i++ 36 | continue 37 | } 38 | 39 | // Find or insert the application in the map 40 | app := apps[md.NameMeta] 41 | if app == nil { 42 | app = &Node{namespace: md.Namespace} 43 | apps[md.NameMeta] = app 44 | } 45 | 46 | // Update the application with the new node information 47 | app.Node, err = node.Pipe( 48 | yaml.FilterFunc(func(object *yaml.RNode) (*yaml.RNode, error) { 49 | return merge2.Merge(app.Node, object, yaml.MergeOptions{}) 50 | }), 51 | 52 | yaml.Tee( 53 | yaml.Lookup("spec", "selector"), 54 | yaml.FilterFunc(func(object *yaml.RNode) (*yaml.RNode, error) { 55 | s := &LabelSelector{} 56 | if err := object.YNode().Decode(s); err != nil { 57 | return nil, err 58 | } 59 | app.selector = s.String() 60 | return nil, nil 61 | }), 62 | ), 63 | 64 | yaml.Tee( 65 | yaml.Lookup("spec", "componentKinds"), 66 | yaml.FilterFunc(func(object *yaml.RNode) (*yaml.RNode, error) { 67 | return nil, object.YNode().Decode(&app.componentKinds) 68 | }), 69 | ), 70 | ) 71 | if err != nil { 72 | return nil, err 73 | } 74 | } 75 | return nodes[:i], nil 76 | } 77 | 78 | type Node struct { 79 | Node *yaml.RNode 80 | namespace string 81 | componentKinds []GroupKind 82 | selector string 83 | } 84 | 85 | // Filter removes all the application resources from the supplied collection. 86 | func (app *Node) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 87 | result := make([]*yaml.RNode, 0, len(nodes)) 88 | for _, node := range nodes { 89 | md, err := node.GetMeta() 90 | if err != nil { 91 | return nil, err 92 | } 93 | 94 | owns, err := app.owns(md, node) 95 | if err != nil { 96 | return nil, err 97 | } 98 | 99 | if !owns { 100 | result = append(result, node) 101 | } 102 | } 103 | return result, nil 104 | } 105 | 106 | func (app *Node) owns(md yaml.ResourceMeta, node *yaml.RNode) (bool, error) { 107 | // The node must be in the same namespace as the application 108 | if md.Namespace != app.namespace { 109 | return false, nil 110 | } 111 | 112 | // The node type must be in the list of application component kinds 113 | var kindMatch bool 114 | for i := range app.componentKinds { 115 | if app.componentKinds[i].Matches(md.TypeMeta) { 116 | kindMatch = true 117 | break 118 | } 119 | } 120 | if !kindMatch { 121 | return false, nil 122 | } 123 | 124 | // The node must match the application label selector 125 | if ok, err := node.MatchesLabelSelector(app.selector); err != nil || !ok { 126 | return false, err 127 | } 128 | 129 | return true, nil 130 | } 131 | -------------------------------------------------------------------------------- /internal/readers/readers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "os/exec" 23 | "path/filepath" 24 | "strings" 25 | 26 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 27 | "sigs.k8s.io/kustomize/kyaml/kio" 28 | "sigs.k8s.io/kustomize/kyaml/yaml" 29 | ) 30 | 31 | // New returns a resource node reader or nil if the input is not recognized. 32 | func New(res any) kio.Reader { 33 | // Construct a new reader based on the input type 34 | switch res := res.(type) { 35 | case *konjurev1beta2.Resource: 36 | return &ResourceReader{Resource: *res} 37 | case *konjurev1beta2.Helm: 38 | return &HelmReader{Helm: *res} 39 | case *konjurev1beta2.Jsonnet: 40 | return NewJsonnetReader(res) 41 | case *konjurev1beta2.Kubernetes: 42 | return &KubernetesReader{Kubernetes: *res} 43 | case *konjurev1beta2.Kustomize: 44 | return &KustomizeReader{Kustomize: *res} 45 | case *konjurev1beta2.Secret: 46 | return &SecretReader{Secret: *res} 47 | case *konjurev1beta2.Git: 48 | return &GitReader{Git: *res} 49 | case *konjurev1beta2.HTTP: 50 | return &HTTPReader{HTTP: *res} 51 | case *konjurev1beta2.File: 52 | return &FileReader{File: *res} 53 | } 54 | return nil 55 | } 56 | 57 | // Executor is function that returns the output of a command. 58 | type Executor func(cmd *exec.Cmd) ([]byte, error) 59 | 60 | // Runtime contains the base configuration for creating `exec.Cmd` instances. 61 | type Runtime struct { 62 | // Bin can be configured to override the default path to the binary. 63 | Bin string 64 | // Executor can be set to change how the command is executed. If left `nil`, 65 | // commands will execute via their `Cmd.Output` function. 66 | Executor Executor 67 | } 68 | 69 | // command returns a new `exec.Cmd` runtime wrapper for the supplied command name. 70 | func (rt *Runtime) command(defBin string) *command { 71 | bin := rt.Bin 72 | if bin == "" { 73 | bin = defBin 74 | } 75 | 76 | return &command{ 77 | Cmd: exec.Command(bin), 78 | Executor: rt.Executor, 79 | } 80 | } 81 | 82 | // command is a runtime wrapper for an `exec.Cmd`. 83 | type command struct { 84 | *exec.Cmd 85 | Executor 86 | } 87 | 88 | // Output invokes the standard `Cmd.Output` function unless there is an explicit 89 | // executor configured to handle execution. 90 | func (cmd *command) Output() ([]byte, error) { 91 | if cmd.Executor != nil { 92 | return cmd.Executor(cmd.Cmd) 93 | } 94 | 95 | return cmd.Cmd.Output() 96 | } 97 | 98 | // Read allows the runtime command to act as a `kio.Reader` assuming the command 99 | // emits YAML manifests to stdout. 100 | func (cmd *command) Read() ([]*yaml.RNode, error) { 101 | out, err := cmd.Output() 102 | if err != nil { 103 | var eerr *exec.ExitError 104 | if errors.As(err, &eerr) { 105 | msg := strings.TrimSpace(string(eerr.Stderr)) 106 | msg = strings.TrimPrefix(msg, "Error: ") 107 | return nil, fmt.Errorf("%s %w: %s", filepath.Base(cmd.Path), err, msg) 108 | } 109 | return nil, err 110 | } 111 | 112 | return kio.FromBytes(out) 113 | } 114 | -------------------------------------------------------------------------------- /internal/readers/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "io" 21 | "path/filepath" 22 | 23 | "sigs.k8s.io/kustomize/kyaml/kio" 24 | "sigs.k8s.io/kustomize/kyaml/yaml" 25 | ) 26 | 27 | // Option is used to configure or decorate a reader. 28 | type Option func(src *yaml.RNode, r kio.Reader) kio.Reader 29 | 30 | // WithDefaultInputStream overrides the default input stream of stdin. 31 | func WithDefaultInputStream(defaultReader io.Reader) Option { 32 | return func(_ *yaml.RNode, r kio.Reader) kio.Reader { 33 | if rr, ok := r.(*ResourceReader); ok && rr.Reader == nil { 34 | rr.Reader = defaultReader 35 | } 36 | return r 37 | } 38 | } 39 | 40 | // WithWorkingDirectory sets the base directory to resolve relative paths against. 41 | func WithWorkingDirectory(dir string) Option { 42 | abs := func(path string) (string, error) { 43 | if filepath.IsAbs(path) { 44 | return filepath.Clean(path), nil 45 | } 46 | return filepath.Join(dir, path), nil 47 | } 48 | 49 | return func(_ *yaml.RNode, r kio.Reader) kio.Reader { 50 | if fr, ok := r.(*FileReader); ok { 51 | fr.Abs = abs 52 | } 53 | return r 54 | } 55 | } 56 | 57 | // WithRecursiveDirectories controls the behavior for traversing directories. 58 | func WithRecursiveDirectories(recurse bool) Option { 59 | return func(_ *yaml.RNode, r kio.Reader) kio.Reader { 60 | if fr, ok := r.(*FileReader); ok { 61 | fr.Recurse = recurse 62 | } 63 | return r 64 | } 65 | } 66 | 67 | // WithKubeconfig controls the default path of the kubeconfig file. 68 | func WithKubeconfig(kubeconfig string) Option { 69 | return func(_ *yaml.RNode, r kio.Reader) kio.Reader { 70 | if kr, ok := r.(*KubernetesReader); ok { 71 | kr.Kubeconfig = kubeconfig 72 | } 73 | return r 74 | } 75 | } 76 | 77 | // WithKubectlExecutor controls the alternate executor for kubectl. 78 | func WithKubectlExecutor(executor Executor) Option { 79 | return func(_ *yaml.RNode, r kio.Reader) kio.Reader { 80 | if kr, ok := r.(*KubernetesReader); ok { 81 | kr.Executor = executor 82 | } 83 | return r 84 | } 85 | } 86 | 87 | // WithKustomizeExecutor controls the alternate executor for kustomize. 88 | func WithKustomizeExecutor(executor Executor) Option { 89 | return func(_ *yaml.RNode, r kio.Reader) kio.Reader { 90 | if kr, ok := r.(*KustomizeReader); ok { 91 | kr.Executor = executor 92 | } 93 | return r 94 | } 95 | } 96 | 97 | // WithDefaultTypes controls the default types to fetch when none are specified. 98 | func WithDefaultTypes(types ...string) Option { 99 | return func(_ *yaml.RNode, r kio.Reader) kio.Reader { 100 | if kr, ok := r.(*KubernetesReader); ok { 101 | kr.DefaultTypes = types 102 | } 103 | return r 104 | } 105 | } 106 | 107 | // WithoutKindExpansion disables specific Konjure kinds from being expanded. 108 | func WithoutKindExpansion(kinds ...string) Option { 109 | return func(src *yaml.RNode, r kio.Reader) kio.Reader { 110 | k := src.GetKind() 111 | for _, kind := range kinds { 112 | if k == kind { 113 | return kio.ResourceNodeSlice{src} 114 | } 115 | } 116 | return r 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /pkg/pipes/file.go: -------------------------------------------------------------------------------- 1 | package pipes 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/fs" 7 | "os" 8 | "path/filepath" 9 | 10 | "sigs.k8s.io/kustomize/kyaml/kio" 11 | "sigs.k8s.io/kustomize/kyaml/kio/kioutil" 12 | "sigs.k8s.io/kustomize/kyaml/yaml" 13 | ) 14 | 15 | type fileReadWriterOptions struct { 16 | // The reader used to parse the file contents. 17 | Reader *kio.ByteReader 18 | // The writer used to encode the file contents. 19 | Writer *kio.ByteWriter 20 | // The default permissions are 0666. 21 | WriteFilePerm os.FileMode 22 | // If non-zero, create the parent directories with these permissions using MkdirAll. 23 | MkdirAllPerm os.FileMode 24 | } 25 | 26 | // FileReadWriterOption represents a configuration option for the FileReader or FileWriter. 27 | type FileReadWriterOption func(opts *fileReadWriterOptions) error 28 | 29 | // FileWriterMkdirAll is an option that allows you to create all the parent directories on write. 30 | func FileWriterMkdirAll(perm os.FileMode) FileReadWriterOption { 31 | return func(opts *fileReadWriterOptions) error { 32 | if opts.Writer == nil { 33 | return fmt.Errorf("mkdirAll requires a FileWriter") 34 | } 35 | opts.MkdirAllPerm = perm 36 | return nil 37 | } 38 | } 39 | 40 | // FileReader is a KIO reader that lazily loads a file. 41 | type FileReader struct { 42 | // The file name to read. 43 | Name string 44 | // The file system to use for resolving file contents (defaults to the OS reader). 45 | FS fs.FS 46 | // Configuration options. 47 | Options []FileReadWriterOption 48 | } 49 | 50 | // Read opens the configured file and reads the contents. 51 | func (r *FileReader) Read() ([]*yaml.RNode, error) { 52 | // TODO Should this open the file so we don't need the whole thing in memory to start? 53 | var data []byte 54 | var err error 55 | if r.FS != nil { 56 | data, err = fs.ReadFile(r.FS, r.Name) 57 | } else { 58 | data, err = os.ReadFile(r.Name) 59 | } 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | // Wrap the reader in an options struct for configuration 65 | opts := fileReadWriterOptions{ 66 | Reader: &kio.ByteReader{ 67 | Reader: bytes.NewReader(data), 68 | SetAnnotations: map[string]string{ 69 | kioutil.PathAnnotation: r.Name, 70 | }, 71 | }, 72 | } 73 | for _, opt := range r.Options { 74 | if err := opt(&opts); err != nil { 75 | return nil, err 76 | } 77 | } 78 | 79 | return opts.Reader.Read() 80 | } 81 | 82 | // FileWriter is a KIO writer that writes nodes to a file. 83 | type FileWriter struct { 84 | // The file name to write. 85 | Name string 86 | // Configuration options. 87 | Options []FileReadWriterOption 88 | } 89 | 90 | // Write overwrites the file with encoded nodes. 91 | func (w *FileWriter) Write(nodes []*yaml.RNode) error { 92 | // TODO Should this open the file so we don't need the whole thing in memory to start? 93 | var buf bytes.Buffer 94 | bw := &kio.ByteWriter{ 95 | Writer: &buf, 96 | ClearAnnotations: []string{ 97 | kioutil.PathAnnotation, 98 | kioutil.LegacyPathAnnotation, 99 | kioutil.IndexAnnotation, 100 | kioutil.LegacyIndexAnnotation, 101 | }, 102 | } 103 | 104 | // Wrap the reader in an options struct for configuration 105 | opts := fileReadWriterOptions{ 106 | Writer: &kio.ByteWriter{ 107 | Writer: &buf, 108 | }, 109 | } 110 | for _, opt := range w.Options { 111 | if err := opt(&opts); err != nil { 112 | return err 113 | } 114 | } 115 | 116 | if err := bw.Write(nodes); err != nil { 117 | return err 118 | } 119 | 120 | if opts.MkdirAllPerm != 0 { 121 | if err := os.MkdirAll(filepath.Dir(w.Name), opts.MkdirAllPerm); err != nil { 122 | return err 123 | } 124 | } 125 | 126 | perm := opts.WriteFilePerm 127 | if perm == 0 { 128 | perm = 0666 129 | } 130 | 131 | return os.WriteFile(w.Name, buf.Bytes(), perm) 132 | } 133 | -------------------------------------------------------------------------------- /internal/readers/kubernetes.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "bufio" 21 | "bytes" 22 | "fmt" 23 | "path" 24 | "strings" 25 | 26 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 27 | "github.com/thestormforge/konjure/pkg/filters" 28 | "sigs.k8s.io/kustomize/kyaml/yaml" 29 | ) 30 | 31 | type KubernetesReader struct { 32 | konjurev1beta2.Kubernetes 33 | Runtime 34 | 35 | // Override the default path to the kubeconfig file. 36 | Kubeconfig string 37 | // Override the default kubeconfig context. 38 | Context string 39 | // The list of default types to use if none are specified. 40 | DefaultTypes []string 41 | } 42 | 43 | func (k *KubernetesReader) Read() ([]*yaml.RNode, error) { 44 | p := &filters.Pipeline{} 45 | 46 | var namespaces []string 47 | if k.AllNamespaces { 48 | namespaces = []string{""} 49 | } else if ns, err := k.namespaces(); err != nil { 50 | return nil, err 51 | } else { 52 | namespaces = ns 53 | } 54 | 55 | types, err := k.types() 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | for _, ns := range namespaces { 61 | cmd := k.command() 62 | cmd.Args = append(cmd.Args, "get") 63 | cmd.Args = append(cmd.Args, "--ignore-not-found") 64 | cmd.Args = append(cmd.Args, "--output", "yaml") 65 | cmd.Args = append(cmd.Args, "--selector", k.Selector) 66 | cmd.Args = append(cmd.Args, "--field-selector", k.FieldSelector) 67 | 68 | if k.AllNamespaces { 69 | cmd.Args = append(cmd.Args, "--all-namespaces") 70 | } 71 | if ns != "" { 72 | cmd.Args = append(cmd.Args, "--namespace", ns) 73 | } 74 | 75 | cmd.Args = append(cmd.Args, strings.Join(types, ",")) 76 | 77 | p.Inputs = append(p.Inputs, cmd) 78 | } 79 | 80 | return p.Read() 81 | } 82 | 83 | func (k *KubernetesReader) command() *command { 84 | cmd := k.Runtime.command("kubectl") 85 | if k.Kubeconfig != "" { 86 | cmd.Args = append(cmd.Args, "--kubeconfig", k.Kubeconfig) 87 | } 88 | if k.Context != "" { 89 | cmd.Args = append(cmd.Args, "--context", k.Context) 90 | } 91 | return cmd 92 | } 93 | 94 | func (k *KubernetesReader) namespaces() ([]string, error) { 95 | if k.Namespace != "" { 96 | return []string{k.Namespace}, nil 97 | } 98 | 99 | if len(k.Namespaces) > 0 { 100 | return k.Namespaces, nil 101 | } 102 | 103 | if k.NamespaceSelector == "" { 104 | return []string{""}, nil 105 | } 106 | 107 | cmd := k.command() 108 | cmd.Args = append(cmd.Args, "get") 109 | cmd.Args = append(cmd.Args, "namespace") 110 | cmd.Args = append(cmd.Args, "--selector", k.NamespaceSelector) 111 | cmd.Args = append(cmd.Args, "--output", "name") 112 | out, err := cmd.Output() 113 | if err != nil { 114 | return nil, err 115 | } 116 | 117 | var namespaces []string 118 | scanner := bufio.NewScanner(bytes.NewReader(out)) 119 | for scanner.Scan() { 120 | namespaces = append(namespaces, path.Base(scanner.Text())) 121 | } 122 | 123 | return namespaces, nil 124 | } 125 | 126 | func (k *KubernetesReader) types() ([]string, error) { 127 | var types []string 128 | for _, t := range k.Types { 129 | if t != "" { 130 | types = append(types, t) 131 | } 132 | } 133 | if len(types) > 0 { 134 | return types, nil 135 | } 136 | 137 | for _, t := range k.DefaultTypes { 138 | if t != "" { 139 | types = append(types, t) 140 | } 141 | } 142 | 143 | if len(types) > 0 { 144 | return types, nil 145 | } 146 | 147 | return nil, fmt.Errorf("no types specified") 148 | } 149 | -------------------------------------------------------------------------------- /pkg/pipes/kubectl.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pipes 18 | 19 | import ( 20 | "context" 21 | "os/exec" 22 | "time" 23 | 24 | "github.com/thestormforge/konjure/pkg/pipes/karg" 25 | ) 26 | 27 | // Kubectl is used for executing `kubectl` as part of a KYAML pipeline. 28 | type Kubectl struct { 29 | // The path the kubectl binary, defaults to `kubectl`. 30 | Bin string 31 | // The path to the kubeconfig. 32 | KubeConfig string 33 | // The context name. 34 | Context string 35 | // The namespace name. 36 | Namespace string 37 | // The length of time to wait before giving up on a single request. 38 | RequestTimeout time.Duration 39 | } 40 | 41 | // Command creates a new executable command with the configured global flags and 42 | // the supplied arguments. 43 | func (k *Kubectl) Command(ctx context.Context, args ...string) *exec.Cmd { 44 | name := k.Bin 45 | if name == "" { 46 | name = "kubectl" 47 | } 48 | 49 | var globalArgs []string 50 | if k.KubeConfig != "" { 51 | globalArgs = append(globalArgs, "--kubeconfig", k.KubeConfig) 52 | } 53 | if k.Context != "" { 54 | globalArgs = append(globalArgs, "--context", k.Context) 55 | } 56 | if k.Namespace != "" { 57 | globalArgs = append(globalArgs, "--namespace", k.Namespace) 58 | } 59 | if k.RequestTimeout != 0 { 60 | globalArgs = append(globalArgs, "--request-timeout", k.RequestTimeout.String()) 61 | } 62 | 63 | return exec.CommandContext(ctx, name, append(globalArgs, args...)...) 64 | } 65 | 66 | // Reader returns a kio.Reader for the specified kubectl arguments. 67 | func (k *Kubectl) Reader(ctx context.Context, args ...string) *ExecReader { 68 | return &ExecReader{ 69 | Cmd: k.Command(ctx, append(args, "--output=yaml")...), 70 | } 71 | } 72 | 73 | // Writer returns a kio.Writer for the specified kubectl arguments. 74 | func (k *Kubectl) Writer(ctx context.Context, args ...string) *ExecWriter { 75 | return &ExecWriter{ 76 | Cmd: k.Command(ctx, append(args, "--filename=-")...), 77 | } 78 | } 79 | 80 | // Get returns a source for getting resources via kubectl. 81 | func (k *Kubectl) Get(ctx context.Context, opts ...karg.GetOption) *ExecReader { 82 | r := k.Reader(ctx, "get") 83 | karg.WithGetOptions(r.Cmd, opts...) 84 | return r 85 | } 86 | 87 | // Create returns a sink for creating resources via kubectl. 88 | func (k *Kubectl) Create(ctx context.Context, opts ...karg.CreateOption) *ExecWriter { 89 | w := k.Writer(ctx, "create") 90 | karg.WithCreateOptions(w.Cmd, opts...) 91 | return w 92 | } 93 | 94 | // Apply returns a sink for applying resources via kubectl. 95 | func (k *Kubectl) Apply(ctx context.Context, opts ...karg.ApplyOption) *ExecWriter { 96 | w := k.Writer(ctx, "apply") 97 | karg.WithApplyOptions(w.Cmd, opts...) 98 | return w 99 | } 100 | 101 | // Delete returns a sink for deleting resources via kubectl. 102 | func (k *Kubectl) Delete(ctx context.Context, opts ...karg.DeleteOption) *ExecWriter { 103 | w := k.Writer(ctx, "delete") 104 | karg.WithDeleteOptions(w.Cmd, opts...) 105 | return w 106 | } 107 | 108 | // Patch returns a sink for patching resources via kubectl. 109 | func (k *Kubectl) Patch(ctx context.Context, opts ...karg.PatchOption) *ExecWriter { 110 | w := k.Writer(ctx, "patch") 111 | karg.WithPatchOptions(w.Cmd, opts...) 112 | return w 113 | } 114 | 115 | // Wait returns a command to wait for conditions via kubectl. 116 | func (k *Kubectl) Wait(ctx context.Context, opts ...karg.WaitOption) *exec.Cmd { 117 | cmd := k.Command(ctx, "wait") 118 | karg.WithWaitOptions(cmd, opts...) 119 | return cmd 120 | } 121 | -------------------------------------------------------------------------------- /pkg/api/core/v1beta2/groupversion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta2 18 | 19 | import ( 20 | "fmt" 21 | 22 | "sigs.k8s.io/kustomize/kyaml/yaml" 23 | ) 24 | 25 | var ( 26 | // Group name for Konjure resources. 27 | Group = "konjure.stormforge.io" 28 | // Version is the current version number for Konjure resources. 29 | Version = "v1beta2" 30 | // APIVersion is the combined group and version string. 31 | APIVersion = Group + "/" + Version 32 | ) 33 | 34 | // NewForType returns a new instance of the typed object identified by the supplied type metadata. 35 | func NewForType(t *yaml.TypeMeta) (any, error) { 36 | if t.APIVersion != APIVersion { 37 | return nil, fmt.Errorf("unknown API version: %s", t.APIVersion) 38 | } 39 | 40 | var result any 41 | switch t.Kind { 42 | case "Resource": 43 | result = new(Resource) 44 | case "Helm": 45 | result = new(Helm) 46 | case "Jsonnet": 47 | result = new(Jsonnet) 48 | case "Kubernetes": 49 | result = new(Kubernetes) 50 | case "Kustomize": 51 | result = new(Kustomize) 52 | case "Secret": 53 | result = new(Secret) 54 | case "Git": 55 | result = new(Git) 56 | case "HTTP": 57 | result = new(HTTP) 58 | case "File": 59 | result = new(File) 60 | default: 61 | return nil, fmt.Errorf("unknown kind: %s", t.Kind) 62 | } 63 | 64 | return result, nil 65 | } 66 | 67 | // GetRNode converts the supplied object to a resource node. 68 | func GetRNode(obj any) (*yaml.RNode, error) { 69 | m := &yaml.ResourceMeta{TypeMeta: yaml.TypeMeta{APIVersion: APIVersion}} 70 | var node any 71 | switch s := obj.(type) { 72 | case *Resource: 73 | m.Kind = "Resource" 74 | node = struct { 75 | Meta *yaml.ResourceMeta `yaml:",inline"` 76 | Spec *Resource `yaml:",inline"` 77 | }{Meta: m, Spec: s} 78 | case *Helm: 79 | m.Kind = "Helm" 80 | node = struct { 81 | Meta *yaml.ResourceMeta `yaml:",inline"` 82 | Spec *Helm `yaml:",inline"` 83 | }{Meta: m, Spec: s} 84 | case *Jsonnet: 85 | m.Kind = "Jsonnet" 86 | node = struct { 87 | Meta *yaml.ResourceMeta `yaml:",inline"` 88 | Spec *Jsonnet `yaml:",inline"` 89 | }{Meta: m, Spec: s} 90 | case *Kubernetes: 91 | m.Kind = "Kubernetes" 92 | node = struct { 93 | Meta *yaml.ResourceMeta `yaml:",inline"` 94 | Spec *Kubernetes `yaml:",inline"` 95 | }{Meta: m, Spec: s} 96 | case *Kustomize: 97 | m.Kind = "Kustomize" 98 | node = struct { 99 | Meta *yaml.ResourceMeta `yaml:",inline"` 100 | Spec *Kustomize `yaml:",inline"` 101 | }{Meta: m, Spec: s} 102 | case *Secret: 103 | m.Kind = "Secret" 104 | node = struct { 105 | Meta *yaml.ResourceMeta `yaml:",inline"` 106 | Spec *Secret `yaml:",inline"` 107 | }{Meta: m, Spec: s} 108 | case *Git: 109 | m.Kind = "Git" 110 | node = struct { 111 | Meta *yaml.ResourceMeta `yaml:",inline"` 112 | Spec *Git `yaml:",inline"` 113 | }{Meta: m, Spec: s} 114 | case *HTTP: 115 | m.Kind = "HTTP" 116 | node = struct { 117 | Meta *yaml.ResourceMeta `yaml:",inline"` 118 | Spec *HTTP `yaml:",inline"` 119 | }{Meta: m, Spec: s} 120 | case *File: 121 | m.Kind = "File" 122 | node = struct { 123 | Meta *yaml.ResourceMeta `yaml:",inline"` 124 | Spec *File `yaml:",inline"` 125 | }{Meta: m, Spec: s} 126 | default: 127 | return nil, fmt.Errorf("unknown type: %T", obj) 128 | } 129 | 130 | data, err := yaml.Marshal(node) 131 | if err != nil { 132 | return nil, err 133 | } 134 | 135 | return yaml.Parse(string(data)) 136 | } 137 | -------------------------------------------------------------------------------- /pkg/filters/fieldpath.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package filters 18 | 19 | import ( 20 | "strconv" 21 | "strings" 22 | "text/template" 23 | 24 | "sigs.k8s.io/kustomize/kyaml/utils" 25 | "sigs.k8s.io/kustomize/kyaml/yaml" 26 | ) 27 | 28 | // FieldPath evaluates a path template using the supplied context and then 29 | // splits it into individual path segments (honoring escaped delimiters). 30 | func FieldPath(p string, data map[string]string) ([]string, error) { 31 | // Evaluate the path as a Go Template 32 | t, err := template.New("path"). 33 | Delims("{", "}"). 34 | Option("missingkey=zero"). 35 | Parse(p) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | var pathBuf strings.Builder 41 | if err := t.Execute(&pathBuf, data); err != nil { 42 | return nil, err 43 | } 44 | 45 | return cleanPath(utils.SmarterPathSplitter(pathBuf.String(), "/")), nil 46 | } 47 | 48 | // SetPath returns a filter that sets the node value at the specified path. 49 | // Note that this path uses "." separators rather then "/". 50 | func SetPath(p string, v *yaml.RNode) yaml.Filter { 51 | path := cleanPath(utils.SmarterPathSplitter(p, ".")) 52 | 53 | var fns []yaml.Filter 54 | if yaml.IsMissingOrNull(v) { 55 | if l := len(path) - 1; l == 0 { 56 | fns = append(fns, 57 | &yaml.FieldClearer{Name: path[l]}, 58 | ) 59 | } else { 60 | fns = append(fns, 61 | &yaml.PathGetter{Path: path[0:l]}, 62 | &yaml.FieldClearer{Name: path[l], IfEmpty: true}, 63 | ) 64 | } 65 | } else { 66 | fns = append(fns, 67 | &yaml.PathGetter{Path: path, Create: v.YNode().Kind}, 68 | &yaml.FieldSetter{Value: v}, 69 | ) 70 | } 71 | 72 | return yaml.FilterFunc(func(object *yaml.RNode) (*yaml.RNode, error) { return object.Pipe(fns...) }) 73 | } 74 | 75 | // SetValues returns a filter that sets the supplied "path=value" specifications 76 | // onto incoming nodes. Use forceString to bypass tagging the node values. 77 | func SetValues(nameValue []string, forceString bool) yaml.Filter { 78 | var fns []yaml.Filter 79 | for _, spec := range nameValue { 80 | fns = append(fns, SetPath(splitPathValue(spec, forceString))) 81 | } 82 | 83 | return yaml.FilterFunc(func(object *yaml.RNode) (*yaml.RNode, error) { return object.Pipe(fns...) }) 84 | } 85 | 86 | // cleanPath removes all empty and white space path elements. 87 | func cleanPath(path []string) []string { 88 | result := make([]string, 0, len(path)) 89 | for _, p := range path { 90 | if p = strings.TrimSpace(p); p != "" { 91 | result = append(result, p) 92 | } 93 | } 94 | if len(result) > 0 { 95 | return result 96 | } 97 | return nil 98 | } 99 | 100 | // splitPathValue splits a "path=value" into a path and an RNode. 101 | func splitPathValue(spec string, st bool) (string, *yaml.RNode) { 102 | p := utils.SmarterPathSplitter(spec, ".") 103 | for i := len(p) - 1; i >= 0; i-- { 104 | path, value, ok := strings.Cut(p[i], "=") 105 | if !ok { 106 | continue 107 | } 108 | 109 | p[i] = path 110 | path = strings.Join(p[0:i+1], ".") 111 | 112 | p[i] = value 113 | value = strings.Join(p[i:], ".") 114 | 115 | node := yaml.NewStringRNode(value) 116 | if st { 117 | return path, node 118 | } 119 | 120 | switch strings.ToLower(value) { 121 | case "true": 122 | node.YNode().Tag = yaml.NodeTagBool 123 | case "false": 124 | node.YNode().Tag = yaml.NodeTagBool 125 | case "null": 126 | node.YNode().Tag = yaml.NodeTagNull 127 | node.YNode().Value = "" 128 | case "0": 129 | node.YNode().Tag = yaml.NodeTagInt 130 | default: 131 | if _, err := strconv.ParseInt(value, 10, 64); err == nil && value[0] != '0' { 132 | node.YNode().Tag = yaml.NodeTagInt 133 | } 134 | } 135 | return path, node 136 | } 137 | return spec, nil 138 | } 139 | -------------------------------------------------------------------------------- /pkg/konjure/resource_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package konjure 18 | 19 | import ( 20 | "bytes" 21 | "encoding/base64" 22 | "encoding/json" 23 | "testing" 24 | 25 | "github.com/stretchr/testify/assert" 26 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 27 | "sigs.k8s.io/kustomize/kyaml/kio" 28 | "sigs.k8s.io/kustomize/kyaml/yaml" 29 | ) 30 | 31 | func TestResource_Read(t *testing.T) { 32 | cases := []struct { 33 | desc string 34 | resource Resource 35 | expected []*yaml.RNode 36 | }{ 37 | { 38 | desc: "helm", 39 | resource: Resource{ 40 | Helm: &konjurev1beta2.Helm{Chart: "test"}, 41 | }, 42 | expected: []*yaml.RNode{mustRNode(&konjurev1beta2.Helm{Chart: "test"})}, 43 | }, 44 | { 45 | desc: "git", 46 | resource: Resource{ 47 | Git: &konjurev1beta2.Git{Repository: "http://example.com/repo"}, 48 | }, 49 | expected: []*yaml.RNode{mustRNode(&konjurev1beta2.Git{Repository: "http://example.com/repo"})}, 50 | }, 51 | } 52 | for _, c := range cases { 53 | t.Run(c.desc, func(t *testing.T) { 54 | actual, err := c.resource.Read() 55 | if assert.NoError(t, err) { 56 | assert.Equal(t, c.expected, actual) 57 | } 58 | }) 59 | } 60 | } 61 | 62 | const testResource = `apiVersion: invalid.example.com/v1 63 | kind: Test 64 | metadata: 65 | name: this-is-a-test 66 | ` 67 | 68 | func TestResource_UnmarshalJSON(t *testing.T) { 69 | cases := []struct { 70 | desc string 71 | rawJSON string 72 | expected Resource 73 | }{ 74 | { 75 | desc: "file string", 76 | rawJSON: `"/this/is/a/test"`, 77 | expected: Resource{ 78 | File: &konjurev1beta2.File{ 79 | Path: "/this/is/a/test", 80 | }, 81 | str: "/this/is/a/test", 82 | }, 83 | }, 84 | { 85 | desc: "file object", 86 | rawJSON: `{"file":{"path":"/this/is/a/test"}}`, 87 | expected: Resource{ 88 | File: &konjurev1beta2.File{ 89 | Path: "/this/is/a/test", 90 | }, 91 | }, 92 | }, 93 | { 94 | desc: "data", 95 | rawJSON: `"data:;base64,` + base64.URLEncoding.EncodeToString([]byte(testResource)) + `"`, 96 | expected: Resource{ 97 | raw: &kio.ByteReader{Reader: bytes.NewReader([]byte(testResource))}, 98 | str: `data:;base64,` + base64.URLEncoding.EncodeToString([]byte(testResource)), 99 | }, 100 | }, 101 | } 102 | for _, c := range cases { 103 | t.Run(c.desc, func(t *testing.T) { 104 | actual := Resource{} 105 | err := json.Unmarshal([]byte(c.rawJSON), &actual) 106 | if assert.NoError(t, err) { 107 | assert.Equal(t, c.expected, actual) 108 | } 109 | }) 110 | } 111 | } 112 | 113 | func TestResource_MarshalJSON(t *testing.T) { 114 | cases := []struct { 115 | desc string 116 | resource Resource 117 | expected string 118 | }{ 119 | { 120 | desc: "file string", 121 | resource: Resource{str: "/this/is/a/test", File: &konjurev1beta2.File{Path: "/this/is/a/test"}}, 122 | expected: `"/this/is/a/test"`, 123 | }, 124 | { 125 | desc: "file object", 126 | resource: Resource{File: &konjurev1beta2.File{Path: "/this/is/a/test"}}, 127 | expected: `"/this/is/a/test"`, 128 | }, 129 | } 130 | for _, c := range cases { 131 | t.Run(c.desc, func(t *testing.T) { 132 | data, err := json.Marshal(&c.resource) 133 | if assert.NoError(t, err) { 134 | assert.JSONEq(t, c.expected, string(data)) 135 | } 136 | }) 137 | } 138 | } 139 | 140 | func TestResource_DeepCopyInto(t *testing.T) { 141 | // Quick sanity test to make sure we keep calm and don't panic 142 | in := Resource{str: "test", File: &konjurev1beta2.File{Path: "test"}} 143 | out := Resource{} 144 | in.DeepCopyInto(&out) 145 | assert.Equal(t, in.str, out.str) 146 | assert.Equal(t, in.File, out.File) 147 | assert.NotSame(t, in.File, out.File) 148 | } 149 | 150 | func mustRNode(obj any) *yaml.RNode { 151 | rn, err := konjurev1beta2.GetRNode(obj) 152 | if err != nil { 153 | panic(err) 154 | } 155 | return rn 156 | } 157 | -------------------------------------------------------------------------------- /pkg/filters/workload.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package filters 18 | 19 | import ( 20 | "sigs.k8s.io/kustomize/kyaml/yaml" 21 | ) 22 | 23 | // WorkloadFilter keeps only workload resources, i.e. those that directly or 24 | // indirectly own pods. For this filter to work, all intermediate resources must 25 | // be present and must have owner metadata specified. 26 | type WorkloadFilter struct { 27 | // Flag indicating if this filter should act as a pass-through. 28 | Enabled bool 29 | // Secondary filter which can be optionally used to accept non-workload resources. 30 | NonWorkloadFilter *ResourceMetaFilter 31 | } 32 | 33 | // Filter keeps all the workload resources. 34 | func (f *WorkloadFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 35 | if !f.Enabled { 36 | return nodes, nil 37 | } 38 | 39 | // TODO https://sdk.operatorframework.io/docs/building-operators/ansible/reference/retroactively-owned-resources/#for-objects-which-are-not-in-the-same-namespace-as-the-owner-cr 40 | 41 | owners := make(map[yaml.ResourceIdentifier]*yaml.ResourceIdentifier, len(nodes)) 42 | pods := make([]yaml.ResourceIdentifier, 0, len(nodes)/3) 43 | unscoped := make(map[yaml.ResourceIdentifier]struct{}) 44 | for _, n := range nodes { 45 | md, err := n.GetMeta() 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | id := md.GetIdentifier() 51 | 52 | // Keep track of pods 53 | if md.APIVersion == "v1" && md.Kind == "Pod" { 54 | pods = append(pods, id) 55 | } 56 | 57 | // Keep track of unscoped nodes 58 | if md.Namespace == "" { 59 | unscoped[id] = struct{}{} 60 | } 61 | 62 | // Index the owner with `controller=true` 63 | if err := n.PipeE( 64 | yaml.Lookup(yaml.MetadataField, "ownerReferences"), 65 | yaml.FilterFunc(func(object *yaml.RNode) (*yaml.RNode, error) { 66 | return nil, object.VisitElements(func(node *yaml.RNode) error { 67 | controller, _ := node.GetFieldValue("controller") 68 | if isController, ok := controller.(bool); !ok || !isController { 69 | return nil 70 | } 71 | 72 | owners[id] = &yaml.ResourceIdentifier{} 73 | return node.YNode().Decode(owners[id]) 74 | }) 75 | })); err != nil { 76 | return nil, err 77 | } 78 | } 79 | 80 | // Find all the distinct workloads by traversing up from the pods 81 | workloads := make(map[yaml.ResourceIdentifier]struct{}, len(nodes)) 82 | for _, pod := range pods { 83 | workload := pod 84 | for { 85 | owner := owners[workload] 86 | if owner == nil { 87 | break 88 | } 89 | 90 | workload = *owner 91 | if _, ok := unscoped[workload]; !ok { 92 | workload.Namespace = pod.Namespace 93 | } 94 | } 95 | workloads[workload] = struct{}{} 96 | } 97 | 98 | // There were no pods found, assume everything we find with a pod template is a workload 99 | if len(pods) == 0 { 100 | for _, n := range nodes { 101 | err := n.PipeE( 102 | Has(yaml.LookupFirstMatch(yaml.ConventionalContainerPaths)), 103 | yaml.FilterFunc(func(object *yaml.RNode) (*yaml.RNode, error) { 104 | md, err := n.GetMeta() 105 | if err != nil { 106 | return nil, err 107 | } 108 | if owners[md.GetIdentifier()] == nil { 109 | workloads[md.GetIdentifier()] = struct{}{} 110 | } 111 | return nil, nil 112 | })) 113 | if err != nil { 114 | return nil, err 115 | } 116 | } 117 | } 118 | 119 | // Filter out the workloads 120 | result := make([]*yaml.RNode, 0, len(workloads)) 121 | for _, n := range nodes { 122 | md, err := n.GetMeta() 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | if _, isWorkload := workloads[md.GetIdentifier()]; isWorkload { 128 | result = append(result, n) 129 | } 130 | } 131 | 132 | // If we have been asked to keep additional workloads, append to the end 133 | if f.NonWorkloadFilter != nil { 134 | extra, err := f.NonWorkloadFilter.Filter(nodes) 135 | if err != nil { 136 | return nil, err 137 | } 138 | result = append(result, extra...) 139 | } 140 | 141 | return result, nil 142 | } 143 | -------------------------------------------------------------------------------- /internal/readers/filter.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "fmt" 21 | 22 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 23 | "sigs.k8s.io/kustomize/kyaml/kio" 24 | "sigs.k8s.io/kustomize/kyaml/yaml" 25 | ) 26 | 27 | // Filter is a KYAML Filter that maps Konjure resource specifications to 28 | // KYAML Readers, then reads and flattens the resulting RNodes into the final 29 | // result. Due to the recursive nature of this filter, the depth (number of 30 | // allowed recursive iterations) must be specified; the default value of 0 is 31 | // effectively a no-op. 32 | type Filter struct { 33 | // The number of iterations to perform when expanding Konjure resources. 34 | Depth int 35 | // Configuration options for the readers. 36 | ReaderOptions []Option 37 | } 38 | 39 | // Filter expands all the Konjure resources using the configured executors. 40 | func (f *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 41 | var err error 42 | 43 | // Recursively expand the nodes to the specified depth 44 | nodes, err = f.expandToDepth(nodes, f.Depth) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | return nodes, nil 50 | } 51 | 52 | // expandToDepth applies the expansion executors up to the specified depth (i.e. a File executor that produces a 53 | // Kustomize resource would be at a depth of 2). 54 | func (f *Filter) expandToDepth(nodes []*yaml.RNode, depth int) ([]*yaml.RNode, error) { 55 | if depth <= 0 { 56 | return nodes, nil 57 | } 58 | 59 | var opts []Option 60 | opts = append(opts, f.ReaderOptions...) 61 | 62 | // Create a new cleaner for this iteration 63 | cleanOpt, doClean := clean() 64 | opts = append(opts, cleanOpt) 65 | defer doClean() 66 | 67 | // Process each of the nodes 68 | result := make([]*yaml.RNode, 0, len(nodes)) 69 | done := true 70 | for _, n := range nodes { 71 | r, err := f.expand(n) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | for _, opt := range opts { 77 | r = opt(n, r) 78 | } 79 | 80 | expanded, err := r.Read() 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | done = done && len(expanded) == 1 && expanded[0] == n 86 | 87 | result = append(result, expanded...) 88 | } 89 | 90 | // Perform another iteration if any of the nodes changed 91 | if !done { 92 | return f.expandToDepth(result, depth-1) 93 | } 94 | return result, nil 95 | } 96 | 97 | // expand returns a reader which can expand the supplied node. If the supplied node 98 | // cannot be expanded, the resulting reader will only produce that node. 99 | func (f *Filter) expand(node *yaml.RNode) (kio.Reader, error) { 100 | m, err := node.GetMeta() 101 | if err != nil { 102 | return nil, err 103 | } 104 | 105 | switch { 106 | 107 | case m.APIVersion == konjurev1beta2.APIVersion: 108 | // Unmarshal the typed Konjure resource and create a reader from it 109 | res, err := konjurev1beta2.NewForType(&m.TypeMeta) 110 | if err != nil { 111 | return nil, err 112 | } 113 | if err := node.YNode().Decode(res); err != nil { 114 | return nil, err 115 | } 116 | r := New(res) 117 | if r == nil { 118 | return nil, fmt.Errorf("unable to read resources from type: %s", m.Kind) 119 | } 120 | return r, nil 121 | 122 | } 123 | 124 | // The default behavior is to just return the node itself 125 | return kio.ResourceNodeSlice{node}, nil 126 | } 127 | 128 | // clean is used to discover readers which implement `cleaner` and invoke their `Clean` function. 129 | func clean() (cleanOpt Option, doClean func()) { 130 | // The cleaner interface can be implemented by readers to implement clean up logic after a filter iteration 131 | type cleaner interface{ Clean() error } 132 | var cleaners []cleaner 133 | 134 | // Accumulate cleaner instances using a reader option 135 | cleanOpt = func(_ *yaml.RNode, r kio.Reader) kio.Reader { 136 | if c, ok := r.(cleaner); ok { 137 | cleaners = append(cleaners, c) 138 | } 139 | return r 140 | } 141 | 142 | // Invoke all the `cleaner.Clean()` functions 143 | doClean = func() { 144 | for _, c := range cleaners { 145 | // TODO This should produce warnings, maybe the errors can be accumulated on the filer itself 146 | _ = c.Clean() 147 | } 148 | } 149 | 150 | return 151 | } 152 | -------------------------------------------------------------------------------- /pkg/filters/fieldpath_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "sigs.k8s.io/kustomize/kyaml/kio" 9 | "sigs.k8s.io/kustomize/kyaml/yaml" 10 | ) 11 | 12 | func TestFieldPath(t *testing.T) { 13 | cases := []struct { 14 | desc string 15 | path string 16 | data map[string]string 17 | expected []string 18 | }{ 19 | { 20 | desc: "empty", 21 | }, 22 | { 23 | desc: "leading slash", 24 | path: "/foo/bar", 25 | expected: []string{"foo", "bar"}, 26 | }, 27 | { 28 | desc: "leading slashes", 29 | path: "////foo/bar", 30 | expected: []string{"foo", "bar"}, 31 | }, 32 | { 33 | desc: "template", 34 | path: "/foo/[bar={.x}]", 35 | data: map[string]string{"x": "test"}, 36 | expected: []string{"foo", "[bar=test]"}, 37 | }, 38 | { 39 | desc: "nested slash", 40 | path: "/foo/[bar=a/b]", 41 | expected: []string{"foo", "[bar=a/b]"}, 42 | }, 43 | } 44 | for _, tc := range cases { 45 | t.Run(tc.desc, func(t *testing.T) { 46 | actual, err := FieldPath(tc.path, tc.data) 47 | if assert.NoError(t, err) { 48 | assert.Equal(t, tc.expected, actual) 49 | } 50 | }) 51 | } 52 | } 53 | 54 | func TestSetValues(t *testing.T) { 55 | cases := []struct { 56 | desc string 57 | inputs []*yaml.RNode 58 | specs []string 59 | forceString bool 60 | expected string 61 | }{ 62 | { 63 | desc: "empty", 64 | }, 65 | { 66 | desc: "simple name", 67 | inputs: []*yaml.RNode{yaml.MustParse(`foo: bar`)}, 68 | specs: []string{"abc=xyz"}, 69 | expected: `# 70 | foo: bar 71 | abc: xyz 72 | `, 73 | }, 74 | { 75 | desc: "indexed name", 76 | inputs: []*yaml.RNode{yaml.MustParse(`foo: bar`)}, 77 | specs: []string{"a.b.[c=d].e=wxyz"}, 78 | expected: `# 79 | foo: bar 80 | a: 81 | b: 82 | - c: d 83 | e: wxyz 84 | `, 85 | }, 86 | { 87 | desc: "no create null", 88 | inputs: []*yaml.RNode{yaml.MustParse(`# 89 | foo: bar 90 | `)}, 91 | specs: []string{"abc=null"}, 92 | expected: `# 93 | foo: bar 94 | `, 95 | }, 96 | { 97 | desc: "unset", 98 | inputs: []*yaml.RNode{yaml.MustParse(`# 99 | foo: bar 100 | abc: xyz 101 | `)}, 102 | specs: []string{"abc=null"}, 103 | expected: `# 104 | foo: bar 105 | `, 106 | }, 107 | { 108 | desc: "unset nested", 109 | inputs: []*yaml.RNode{yaml.MustParse(`# 110 | foo: bar 111 | abc: 112 | xyz: zyx 113 | `)}, 114 | specs: []string{"abc.xyz=null"}, 115 | expected: `# 116 | foo: bar 117 | abc: {} 118 | `, 119 | }, 120 | } 121 | for _, tc := range cases { 122 | t.Run(tc.desc, func(t *testing.T) { 123 | var buf strings.Builder 124 | err := kio.Pipeline{ 125 | Inputs: []kio.Reader{kio.ResourceNodeSlice(tc.inputs)}, 126 | Filters: []kio.Filter{kio.FilterAll(SetValues(tc.specs, tc.forceString))}, 127 | Outputs: []kio.Writer{kio.ByteWriter{Writer: &buf}}, 128 | }.Execute() 129 | if assert.NoError(t, err, "Pipeline failed") { 130 | assert.YAMLEq(t, tc.expected, buf.String()) 131 | } 132 | }) 133 | } 134 | } 135 | 136 | func TestSplitPathValue(t *testing.T) { 137 | cases := []struct { 138 | desc string 139 | spec string 140 | expectedPath string 141 | expectedValue *yaml.RNode 142 | }{ 143 | { 144 | desc: "empty", 145 | }, 146 | { 147 | desc: "simple name string value", 148 | spec: "name=value", 149 | expectedPath: "name", 150 | expectedValue: yaml.NewStringRNode("value"), 151 | }, 152 | { 153 | desc: "nested name string value", 154 | spec: "nom.name=value", 155 | expectedPath: "nom.name", 156 | expectedValue: yaml.NewStringRNode("value"), 157 | }, 158 | { 159 | desc: "indexed name string value", 160 | spec: "nom.name.[foo.bar=xyz].foobar=value", 161 | expectedPath: "nom.name.[foo.bar=xyz].foobar", 162 | expectedValue: yaml.NewStringRNode("value"), 163 | }, 164 | { 165 | desc: "string value with delimiter", 166 | spec: "name=Value. Such value.", 167 | expectedPath: "name", 168 | expectedValue: yaml.NewStringRNode("Value. Such value."), 169 | }, 170 | { 171 | desc: "empty string value", 172 | spec: "name=", 173 | expectedPath: "name", 174 | expectedValue: yaml.NewStringRNode(""), 175 | }, 176 | { 177 | desc: "null value", 178 | spec: "name=null", 179 | expectedPath: "name", 180 | expectedValue: yaml.NewRNode(&yaml.Node{Kind: yaml.ScalarNode, Tag: yaml.NodeTagNull}), 181 | }, 182 | } 183 | for _, tc := range cases { 184 | t.Run(tc.desc, func(t *testing.T) { 185 | actualPath, actualValue := splitPathValue(tc.spec, false) 186 | assert.Equal(t, tc.expectedPath, actualPath) 187 | assert.Equal(t, tc.expectedValue, actualValue) 188 | }) 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /internal/command/jsonnet.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package command 18 | 19 | import ( 20 | "github.com/spf13/cobra" 21 | "github.com/thestormforge/konjure/internal/readers" 22 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 23 | "github.com/thestormforge/konjure/pkg/konjure" 24 | "sigs.k8s.io/kustomize/kyaml/kio" 25 | ) 26 | 27 | func NewJsonnetCommand() *cobra.Command { 28 | f := jsonnetFlags{} 29 | 30 | cmd := &cobra.Command{ 31 | Use: "jsonnet", 32 | Short: "Evaluate a Jsonnet program", 33 | Args: cobra.ExactArgs(1), 34 | PreRun: f.preRun, 35 | RunE: func(cmd *cobra.Command, args []string) error { 36 | return kio.Pipeline{ 37 | Inputs: []kio.Reader{readers.NewJsonnetReader(&f.Jsonnet)}, 38 | Outputs: []kio.Writer{&konjure.Writer{Writer: cmd.OutOrStdout()}}, 39 | }.Execute() 40 | }, 41 | } 42 | 43 | cmd.Flags().BoolVarP(&f.execute, "exec", "e", false, "treat argument as code") 44 | cmd.Flags().StringToStringVarP(&f.externalStringVariables, "ext-str", "V", nil, "provide external variable as a string") 45 | cmd.Flags().StringToStringVar(&f.externalStringFileVariables, "ext-str-file", nil, "provide external variable as a string from the file") 46 | cmd.Flags().StringToStringVar(&f.externalCodeVariables, "ext-code", nil, "provide external variable as Jsonnet code") 47 | cmd.Flags().StringToStringVar(&f.externalCodeFileVariables, "ext-code-file", nil, "provide external variable as Jsonnet code from the file") 48 | cmd.Flags().StringToStringVarP(&f.topLevelStringArguments, "tla-str", "A", nil, "provide top-level argument as a string") 49 | cmd.Flags().StringToStringVar(&f.topLevelStringFileArguments, "tla-str-file", nil, "provide top-level argument as a string from the file") 50 | cmd.Flags().StringToStringVar(&f.topLevelCodeArguments, "tla-code", nil, "provide top-level argument as Jsonnet code") 51 | cmd.Flags().StringToStringVar(&f.topLevelCodeFileArguments, "tla-code-file", nil, "provide top-level argument as Jsonnet code from the file") 52 | cmd.Flags().StringArrayVarP(&f.JsonnetPath, "jpath", "J", nil, "specify an additional library search directory") 53 | cmd.Flags().StringVar(&f.JsonnetBundlerPackageHome, "jsonnetpkg-home", "", "the directory used to cache packages in") 54 | cmd.Flags().BoolVar(&f.JsonnetBundlerRefresh, "jsonnetpkg-refresh", false, "force update dependencies") 55 | 56 | return cmd 57 | } 58 | 59 | type jsonnetFlags struct { 60 | konjurev1beta2.Jsonnet 61 | execute bool 62 | externalStringVariables map[string]string 63 | externalStringFileVariables map[string]string 64 | externalCodeVariables map[string]string 65 | externalCodeFileVariables map[string]string 66 | topLevelStringArguments map[string]string 67 | topLevelStringFileArguments map[string]string 68 | topLevelCodeArguments map[string]string 69 | topLevelCodeFileArguments map[string]string 70 | } 71 | 72 | func (f *jsonnetFlags) preRun(_ *cobra.Command, args []string) { 73 | if f.execute { 74 | f.Code = args[0] 75 | } else { 76 | f.Filename = args[0] 77 | } 78 | 79 | for k, v := range f.externalStringVariables { 80 | f.ExternalVariables = append(f.ExternalVariables, konjurev1beta2.JsonnetParameter{Name: k, String: v}) 81 | } 82 | for k, v := range f.externalStringFileVariables { 83 | f.ExternalVariables = append(f.ExternalVariables, konjurev1beta2.JsonnetParameter{Name: k, StringFile: v}) 84 | } 85 | for k, v := range f.externalCodeVariables { 86 | f.ExternalVariables = append(f.ExternalVariables, konjurev1beta2.JsonnetParameter{Name: k, Code: v}) 87 | } 88 | for k, v := range f.externalCodeFileVariables { 89 | f.ExternalVariables = append(f.ExternalVariables, konjurev1beta2.JsonnetParameter{Name: k, CodeFile: v}) 90 | } 91 | 92 | for k, v := range f.topLevelStringArguments { 93 | f.TopLevelArguments = append(f.TopLevelArguments, konjurev1beta2.JsonnetParameter{Name: k, String: v}) 94 | } 95 | for k, v := range f.topLevelStringFileArguments { 96 | f.TopLevelArguments = append(f.TopLevelArguments, konjurev1beta2.JsonnetParameter{Name: k, StringFile: v}) 97 | } 98 | for k, v := range f.topLevelCodeArguments { 99 | f.TopLevelArguments = append(f.TopLevelArguments, konjurev1beta2.JsonnetParameter{Name: k, Code: v}) 100 | } 101 | for k, v := range f.topLevelCodeFileArguments { 102 | f.TopLevelArguments = append(f.TopLevelArguments, konjurev1beta2.JsonnetParameter{Name: k, CodeFile: v}) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /internal/command/root.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package command 18 | 19 | import ( 20 | "os" 21 | 22 | "github.com/spf13/cobra" 23 | "github.com/thestormforge/konjure/pkg/konjure" 24 | "sigs.k8s.io/kustomize/kyaml/kio" 25 | "sigs.k8s.io/kustomize/kyaml/kio/filters" 26 | "sigs.k8s.io/kustomize/kyaml/kio/kioutil" 27 | ) 28 | 29 | func NewRootCommand(version, refspec, date string) *cobra.Command { 30 | r := konjure.Resources{} 31 | f := &konjure.Filter{} 32 | w := &konjure.Writer{} 33 | 34 | cmd := &cobra.Command{ 35 | Use: "konjure INPUT...", 36 | Short: "Manifest, appear!", 37 | Version: version, 38 | SilenceUsage: true, 39 | TraverseChildren: true, 40 | ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { 41 | return []string{"yaml", "yml", "json"}, cobra.ShellCompDirectiveFilterFileExt 42 | }, 43 | Annotations: map[string]string{ 44 | "BuildRefspec": refspec, 45 | "BuildDate": date, 46 | }, 47 | PreRunE: func(cmd *cobra.Command, args []string) (err error) { 48 | w.Writer = cmd.OutOrStdout() 49 | f.DefaultReader = cmd.InOrStdin() 50 | 51 | if len(args) > 0 { 52 | r = append(r, konjure.NewResource(args...)) 53 | } else { 54 | r = append(r, konjure.NewResource("-")) 55 | } 56 | 57 | f.WorkingDirectory, err = os.Getwd() 58 | 59 | if !w.KeepReaderAnnotations { 60 | w.ClearAnnotations = append(w.ClearAnnotations, 61 | kioutil.PathAnnotation, 62 | kioutil.LegacyPathAnnotation, 63 | filters.FmtAnnotation, 64 | ) 65 | } 66 | 67 | return 68 | }, 69 | RunE: func(cmd *cobra.Command, args []string) error { 70 | return kio.Pipeline{ 71 | Inputs: []kio.Reader{r}, 72 | Filters: []kio.Filter{f}, 73 | Outputs: []kio.Writer{w}, 74 | ContinueOnEmptyResult: true, 75 | }.Execute() 76 | }, 77 | } 78 | 79 | cmd.Flags().IntVarP(&f.Depth, "depth", "d", 100, "limit the number of times expansion can happen") 80 | cmd.Flags().StringVarP(&f.LabelSelector, "selector", "l", "", "label query to filter on") 81 | cmd.Flags().StringVar(&f.Kind, "kind", "", "keep only resource matching the specified kind") 82 | cmd.Flags().BoolVar(&f.KeepStatus, "keep-status", false, "retain status fields, if present") 83 | cmd.Flags().BoolVar(&f.KeepComments, "keep-comments", true, "retain YAML comments") 84 | cmd.Flags().BoolVar(&f.ResetStyle, "reset-style", false, "reset YAML style") 85 | cmd.Flags().BoolVar(&f.Format, "format", false, "format output to Kubernetes conventions") 86 | cmd.Flags().BoolVar(&w.RestoreVerticalWhiteSpace, "vws", false, "attempt to restore vertical white space") 87 | cmd.Flags().BoolVarP(&f.RecursiveDirectories, "recurse", "r", false, "recursively process directories") 88 | cmd.Flags().StringVar(&f.Kubeconfig, "kubeconfig", "", "path to the kubeconfig file") 89 | cmd.Flags().StringVarP(&w.Format, "output", "o", "yaml", "set the output format (yaml, json, ndjson, env, name, columns=, csv=, template=)") 90 | cmd.Flags().BoolVar(&w.KeepReaderAnnotations, "keep-annotations", false, "retain annotations used for processing") 91 | cmd.Flags().BoolVar(&f.Sort, "sort", false, "sort output prior to writing") 92 | cmd.Flags().BoolVar(&f.Reverse, "reverse", false, "reverse sort output prior to writing") 93 | cmd.Flags().StringSliceVar(&f.DoNotExpand, "do-not-expand", nil, "do not expand Konjure kinds (Resource, Helm, Jsonnet, Kubernetes, Kustomize, Secret, Git, HTTP, File)") 94 | cmd.Flags().BoolVar(&f.ApplicationFilter.Enabled, "apps", false, "transform output to application definitions") 95 | cmd.Flags().StringSliceVar(&f.ApplicationFilter.ApplicationNameLabels, "application-name-label", nil, "label to use for application names") 96 | cmd.Flags().BoolVar(&f.WorkloadFilter.Enabled, "workloads", false, "keep only workload resources") 97 | 98 | _ = cmd.Flags().MarkHidden("do-not-expand") 99 | 100 | _ = cmd.Flags().MarkHidden("apps") // TODO This is "early access" 101 | _ = cmd.Flags().MarkHidden("application-name-label") // TODO This is "early access" 102 | _ = cmd.Flags().MarkHidden("workloads") // TODO This is "early access" 103 | _ = cmd.Flags().MarkHidden("vws") // TODO This is "early access" / "somewhat unstable" 104 | 105 | cmd.AddCommand( 106 | NewHelmCommand(), 107 | NewHelmValuesCommand(), 108 | NewJsonnetCommand(), 109 | NewSecretCommand(), 110 | ) 111 | 112 | return cmd 113 | } 114 | -------------------------------------------------------------------------------- /pkg/konjure/filter.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package konjure 18 | 19 | import ( 20 | "io" 21 | "os/exec" 22 | 23 | "github.com/thestormforge/konjure/internal/readers" 24 | "github.com/thestormforge/konjure/pkg/filters" 25 | "sigs.k8s.io/kustomize/kyaml/kio" 26 | kiofilters "sigs.k8s.io/kustomize/kyaml/kio/filters" 27 | "sigs.k8s.io/kustomize/kyaml/yaml" 28 | ) 29 | 30 | // Filter replaces Konjure resources with the expanded resources they represent. 31 | type Filter struct { 32 | // The number of times to recursively filter the resource list. 33 | Depth int 34 | // The default reader to use, defaults to stdin. 35 | DefaultReader io.Reader 36 | // Filter used to reduce the output to application definitions. 37 | ApplicationFilter filters.ApplicationFilter 38 | // Filter used to reduce the output to workloads. 39 | WorkloadFilter filters.WorkloadFilter 40 | // Filter to determine which resources are retained. 41 | filters.ResourceMetaFilter 42 | // Flag indicating that status fields should not be stripped. 43 | KeepStatus bool 44 | // Flag indicating that comments should not be stripped. 45 | KeepComments bool 46 | // Flag indicating that style should be reset. 47 | ResetStyle bool 48 | // Flag indicating that output should be formatted. 49 | Format bool 50 | // Flag indicating that output should be sorted. 51 | Sort bool 52 | // Flag indicating that output should be reverse sorted (implies sort=true). 53 | Reverse bool 54 | // The explicit working directory used to resolve relative paths. 55 | WorkingDirectory string 56 | // Flag indicating we can process directories recursively. 57 | RecursiveDirectories bool 58 | // Kinds which should not be expanded (e.g. "Kustomize"). 59 | DoNotExpand []string 60 | // Override the default path to the kubeconfig file. 61 | Kubeconfig string 62 | // Override the default types used when fetching Kubernetes resources. 63 | KubernetesTypes []string 64 | // Override the default Kubectl executor. 65 | KubectlExecutor func(cmd *exec.Cmd) ([]byte, error) 66 | // Override the default Kustomize executor. 67 | KustomizeExecutor func(cmd *exec.Cmd) ([]byte, error) 68 | } 69 | 70 | // Filter evaluates Konjure resources according to the filter configuration. 71 | func (f *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 72 | defaultTypes := f.KubernetesTypes 73 | if len(defaultTypes) == 0 { 74 | // This represents the original set of default types from early StormForge products 75 | defaultTypes = append(defaultTypes, "deployments", "statefulsets", "configmaps") 76 | } 77 | 78 | if f.WorkloadFilter.Enabled { 79 | // Include the built-in workload types (and intermediaries necessary for detection to work) 80 | defaultTypes = appendDistinct(defaultTypes, "daemonsets", "deployments", "statefulsets", "replicasets", "cronjobs", "pods") 81 | } 82 | 83 | p := &filters.Pipeline{ 84 | Inputs: []kio.Reader{kio.ResourceNodeSlice(nodes)}, 85 | Filters: []kio.Filter{ 86 | &readers.Filter{ 87 | Depth: f.Depth, 88 | ReaderOptions: []readers.Option{ 89 | readers.WithDefaultInputStream(f.DefaultReader), 90 | readers.WithWorkingDirectory(f.WorkingDirectory), 91 | readers.WithRecursiveDirectories(f.RecursiveDirectories), 92 | readers.WithKubeconfig(f.Kubeconfig), 93 | readers.WithKubectlExecutor(f.KubectlExecutor), 94 | readers.WithKustomizeExecutor(f.KustomizeExecutor), 95 | readers.WithDefaultTypes(defaultTypes...), 96 | readers.WithoutKindExpansion(f.DoNotExpand...), 97 | }, 98 | }, 99 | 100 | &f.ApplicationFilter, 101 | &f.WorkloadFilter, 102 | &f.ResourceMetaFilter, 103 | }, 104 | } 105 | 106 | if !f.KeepStatus { 107 | p.Filters = append(p.Filters, kio.FilterAll(yaml.Clear("status"))) 108 | } 109 | 110 | if !f.KeepComments { 111 | p.Filters = append(p.Filters, &kiofilters.StripCommentsFilter{}) 112 | } 113 | 114 | if f.ResetStyle { 115 | p.Filters = append(p.Filters, kio.FilterAll(filters.ResetStyle())) 116 | } 117 | 118 | if f.Format { 119 | p.Filters = append(p.Filters, &kiofilters.FormatFilter{}) 120 | } 121 | 122 | if f.Reverse { 123 | p.Filters = append(p.Filters, filters.UninstallOrder()) 124 | } else if f.Sort { 125 | p.Filters = append(p.Filters, filters.InstallOrder()) 126 | } 127 | 128 | return p.Read() 129 | } 130 | 131 | func appendDistinct(values []string, more ...string) []string { 132 | contains := make(map[string]struct{}, len(values)+len(more)) 133 | for _, v := range values { 134 | contains[v] = struct{}{} 135 | } 136 | for _, m := range more { 137 | if _, ok := contains[m]; !ok { 138 | contains[m] = struct{}{} 139 | values = append(values, m) 140 | } 141 | } 142 | return values 143 | } 144 | -------------------------------------------------------------------------------- /pkg/pipes/pipes.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pipes 18 | 19 | import ( 20 | "bytes" 21 | "encoding/json" 22 | "fmt" 23 | "text/template" 24 | 25 | "github.com/thestormforge/konjure/pkg/filters" 26 | "sigs.k8s.io/kustomize/kyaml/kio" 27 | "sigs.k8s.io/kustomize/kyaml/yaml" 28 | ) 29 | 30 | // ReaderFunc is an adapter to allow the use of ordinary functions as a kio.Reader. 31 | type ReaderFunc func() ([]*yaml.RNode, error) 32 | 33 | // Read evaluates the typed function. 34 | func (r ReaderFunc) Read() ([]*yaml.RNode, error) { return r() } 35 | 36 | // ReadOneFunc is an adapter to allow the use of single node returning functions as a kio.Reader. 37 | type ReadOneFunc func() (*yaml.RNode, error) 38 | 39 | // Read evaluates the typed function and wraps the resulting non-nil node. 40 | func (r ReadOneFunc) Read() ([]*yaml.RNode, error) { 41 | node, err := r() 42 | if node != nil { 43 | return []*yaml.RNode{node}, err 44 | } 45 | return nil, err 46 | } 47 | 48 | // ErrorReader is an adapter to allow the use of an error as a kio.Reader. 49 | type ErrorReader struct{ Err error } 50 | 51 | // Reader returns the wrapped failure. 52 | func (r ErrorReader) Read() ([]*yaml.RNode, error) { return nil, r.Err } 53 | 54 | // Encode returns a reader over the YAML encoding of the specified values. 55 | func Encode(values ...any) kio.Reader { 56 | return encodingReader(values) 57 | } 58 | 59 | // encodingReader is an adapter to allow arbitrary values to be used as a kio.Reader. 60 | type encodingReader []any 61 | 62 | // Read encodes the configured values. 63 | func (r encodingReader) Read() ([]*yaml.RNode, error) { 64 | result := make([]*yaml.RNode, len(r)) 65 | for i := range r { 66 | result[i] = yaml.NewRNode(&yaml.Node{}) 67 | if err := result[i].YNode().Encode(r[i]); err != nil { 68 | return nil, err 69 | } 70 | } 71 | return result, nil 72 | } 73 | 74 | // EncodeJSON returns a reader over the JSON encoding of the specified values. 75 | func EncodeJSON(values ...any) kio.Reader { 76 | return encodingJSONReader(values) 77 | } 78 | 79 | type encodingJSONReader []any 80 | 81 | func (r encodingJSONReader) Read() ([]*yaml.RNode, error) { 82 | nodes := make([]*yaml.RNode, len(r)) 83 | for i := range r { 84 | nodes[i] = yaml.NewRNode(&yaml.Node{}) 85 | var buf bytes.Buffer 86 | if err := json.NewEncoder(&buf).Encode(r[i]); err != nil { 87 | return nil, err 88 | } else if err := yaml.NewDecoder(&buf).Decode(nodes[i].YNode()); err != nil { 89 | return nil, err 90 | } 91 | } 92 | return filters.FilterAll(filters.ResetStyle()).Filter(nodes) 93 | } 94 | 95 | // Decode returns a writer over the YAML decoding (one per resource document). 96 | func Decode(values ...any) kio.Writer { 97 | return &decodingWriter{Values: values} 98 | } 99 | 100 | // decodingWriter is an adapter to allow arbitrary values to be used as a kio.Writer. 101 | type decodingWriter struct{ Values []any } 102 | 103 | // Write decodes the incoming nodes. 104 | func (w *decodingWriter) Write(nodes []*yaml.RNode) error { 105 | if len(nodes) != len(w.Values) { 106 | return fmt.Errorf("document count mismatch, expected %d, got %d", len(w.Values), len(nodes)) 107 | } 108 | for i := range w.Values { 109 | if err := nodes[i].YNode().Decode(w.Values[i]); err != nil { 110 | return err 111 | } 112 | } 113 | return nil 114 | } 115 | 116 | // DecodeJSON returns a writer over the JSON decoding of the YAML (one per resource document). 117 | func DecodeJSON(values ...any) kio.Writer { 118 | return &decodingJSONWriter{Values: values} 119 | } 120 | 121 | // decodingWriter is an adapter to allow arbitrary values to be used as a kio.Writer. 122 | type decodingJSONWriter struct{ Values []any } 123 | 124 | // Write decodes the incoming nodes as JSON. 125 | func (w *decodingJSONWriter) Write(nodes []*yaml.RNode) error { 126 | if len(nodes) != len(w.Values) { 127 | return fmt.Errorf("document count mismatch, expected %d, got %d", len(w.Values), len(nodes)) 128 | } 129 | for i := range w.Values { 130 | // WARNING: This only works with mapping and sequence nodes 131 | data, err := nodes[i].MarshalJSON() 132 | if err != nil { 133 | return err 134 | } 135 | if err := json.Unmarshal(data, w.Values[i]); err != nil { 136 | return err 137 | } 138 | } 139 | return nil 140 | } 141 | 142 | // TemplateReader is a KYAML reader that consumes YAML from a Go template. 143 | type TemplateReader struct { 144 | // The template to execute. 145 | Template *template.Template 146 | // The data for the template. 147 | Data any 148 | } 149 | 150 | // Read executes the supplied template and parses the output as a YAML document stream. 151 | func (c *TemplateReader) Read() ([]*yaml.RNode, error) { 152 | var buf bytes.Buffer 153 | if err := c.Template.Execute(&buf, c.Data); err != nil { 154 | return nil, err 155 | } 156 | 157 | return (&kio.ByteReader{ 158 | Reader: &buf, 159 | }).Read() 160 | } 161 | -------------------------------------------------------------------------------- /pkg/filters/resourcemeta.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package filters 18 | 19 | import ( 20 | "regexp" 21 | "strings" 22 | 23 | "sigs.k8s.io/kustomize/kyaml/openapi" 24 | "sigs.k8s.io/kustomize/kyaml/yaml" 25 | ) 26 | 27 | // ResourceMetaFilter filters nodes based on their resource metadata using regular 28 | // expressions or Kubernetes selectors. 29 | type ResourceMetaFilter struct { 30 | // Regular expression matching the group. 31 | Group string `json:"group,omitempty" yaml:"group,omitempty"` 32 | // Regular expression matching the version. 33 | Version string `json:"version,omitempty" yaml:"version,omitempty"` 34 | // Regular expression matching the kind. 35 | Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` 36 | // Regular expression matching the namespace. 37 | Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` 38 | // Regular expression matching the name. 39 | Name string `json:"name,omitempty" yaml:"name,omitempty"` 40 | // Kubernetes selector matching labels. 41 | LabelSelector string `json:"labelSelector,omitempty" yaml:"labelSelector,omitempty"` 42 | // Kubernetes selector matching annotations 43 | AnnotationSelector string `json:"annotationSelector,omitempty" yaml:"annotationSelector,omitempty"` 44 | 45 | // Invert the matching behavior (i.e. keep non-matching nodes). 46 | InvertMatch bool `json:"invertMatch,omitempty" yaml:"invertMatch,omitempty"` 47 | } 48 | 49 | func (f *ResourceMetaFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 50 | m, err := newMetaMatcher(f) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | if m == nil && f.LabelSelector == "" && f.AnnotationSelector == "" { 56 | return nodes, nil 57 | } 58 | 59 | result := make([]*yaml.RNode, 0, len(nodes)) 60 | for _, n := range nodes { 61 | if m != nil { 62 | if meta, err := n.GetMeta(); err != nil { 63 | return nil, err 64 | } else if m.matchesMeta(meta) == f.InvertMatch { 65 | continue 66 | } 67 | } 68 | 69 | if f.LabelSelector != "" { 70 | if matched, err := n.MatchesLabelSelector(f.LabelSelector); err != nil { 71 | return nil, err 72 | } else if matched == f.InvertMatch { 73 | continue 74 | } 75 | } 76 | 77 | if f.AnnotationSelector != "" { 78 | if matched, err := n.MatchesAnnotationSelector(f.AnnotationSelector); err != nil { 79 | return nil, err 80 | } else if matched == f.InvertMatch { 81 | continue 82 | } 83 | } 84 | 85 | result = append(result, n) 86 | } 87 | 88 | return result, nil 89 | } 90 | 91 | type metaMatcher struct { 92 | namespaceRegex *regexp.Regexp 93 | nameRegex *regexp.Regexp 94 | groupRegex *regexp.Regexp 95 | versionRegex *regexp.Regexp 96 | kindRegex *regexp.Regexp 97 | } 98 | 99 | func newMetaMatcher(g *ResourceMetaFilter) (m *metaMatcher, err error) { 100 | m = &metaMatcher{} 101 | notEmpty := false 102 | 103 | m.namespaceRegex, err = compileAnchored(g.Namespace) 104 | if err != nil { 105 | return nil, err 106 | } 107 | notEmpty = notEmpty || m.namespaceRegex != nil 108 | 109 | m.nameRegex, err = compileAnchored(g.Name) 110 | if err != nil { 111 | return nil, err 112 | } 113 | notEmpty = notEmpty || m.nameRegex != nil 114 | 115 | m.groupRegex, err = compileAnchored(g.Group) 116 | if err != nil { 117 | return nil, err 118 | } 119 | notEmpty = notEmpty || m.groupRegex != nil 120 | 121 | m.versionRegex, err = compileAnchored(g.Version) 122 | if err != nil { 123 | return nil, err 124 | } 125 | notEmpty = notEmpty || m.versionRegex != nil 126 | 127 | m.kindRegex, err = compileAnchored(g.Kind) 128 | if err != nil { 129 | return nil, err 130 | } 131 | notEmpty = notEmpty || m.kindRegex != nil 132 | 133 | if notEmpty { 134 | return m, nil 135 | } 136 | return nil, nil 137 | } 138 | 139 | func (m *metaMatcher) matchesMeta(meta yaml.ResourceMeta) bool { 140 | if m.namespaceRegex != nil && !m.namespaceRegex.MatchString(meta.Namespace) { 141 | return false 142 | } 143 | if m.nameRegex != nil && !m.nameRegex.MatchString(meta.Name) { 144 | return false 145 | } 146 | 147 | if m.groupRegex != nil || m.versionRegex != nil { 148 | group, version := "", meta.APIVersion 149 | if pos := strings.Index(version, "/"); pos >= 0 { 150 | group, version = version[0:pos], version[pos+1:] 151 | } 152 | 153 | if m.groupRegex != nil && !m.groupRegex.MatchString(group) { 154 | return false 155 | } 156 | 157 | if m.versionRegex != nil && !m.versionRegex.MatchString(version) { 158 | return false 159 | } 160 | } 161 | 162 | if m.kindRegex != nil && !m.kindRegex.MatchString(meta.Kind) { 163 | return false 164 | } 165 | 166 | return true 167 | } 168 | 169 | func compileAnchored(pattern string) (*regexp.Regexp, error) { 170 | if pattern == "" { 171 | return nil, nil 172 | } 173 | return regexp.Compile("^(?:" + pattern + ")$") 174 | } 175 | 176 | // SetNamespace is like `SetK8sNamespace` except it skips cluster scoped resources. 177 | func SetNamespace(namespace string) yaml.Filter { 178 | return yaml.FilterFunc(func(node *yaml.RNode) (*yaml.RNode, error) { 179 | return node.Pipe( 180 | yaml.Tee( 181 | yaml.FilterFunc(func(node *yaml.RNode) (*yaml.RNode, error) { 182 | md, err := node.GetMeta() 183 | if err != nil { 184 | return nil, err 185 | } 186 | 187 | if openapi.IsCertainlyClusterScoped(md.TypeMeta) { 188 | return nil, nil 189 | } 190 | 191 | return node, nil 192 | }), 193 | yaml.SetK8sNamespace(namespace), 194 | ), 195 | ) 196 | }) 197 | } 198 | -------------------------------------------------------------------------------- /internal/readers/file.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "io/fs" 23 | "os" 24 | "path/filepath" 25 | "strings" 26 | 27 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 28 | "sigs.k8s.io/kustomize/kyaml/kio" 29 | "sigs.k8s.io/kustomize/kyaml/kio/kioutil" 30 | "sigs.k8s.io/kustomize/kyaml/yaml" 31 | ) 32 | 33 | type FileReader struct { 34 | konjurev1beta2.File 35 | 36 | // Flag indicating we are allowed to recurse into directories. 37 | Recurse bool 38 | // Function used to determine an absolute path. 39 | Abs func(path string) (string, error) 40 | } 41 | 42 | func (r *FileReader) Read() ([]*yaml.RNode, error) { 43 | root, err := r.root() 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | var result []*yaml.RNode 49 | err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { 50 | // Just bubble walk errors back up 51 | if err != nil { 52 | // TODO Check the annotations on the File object to see if there is any context we can add to the error 53 | return err 54 | } 55 | 56 | if info.IsDir() { 57 | // Determine if we are allowed to recurse into the directory 58 | if !r.Recurse && path != root { 59 | return filepath.SkipDir 60 | } 61 | 62 | // See if the directory itself expands 63 | if result, err = r.readDir(result, path); err != nil { 64 | return err 65 | } 66 | 67 | return nil 68 | } 69 | 70 | // Try to figure out what to do based on the file extension 71 | switch strings.ToLower(filepath.Ext(path)) { 72 | 73 | case ".jsonnet": 74 | n, err := konjurev1beta2.GetRNode(&konjurev1beta2.Jsonnet{Filename: path}) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | result = append(result, n) 80 | 81 | case ".json", ".yaml", ".yml", "": 82 | // Just read the data in, assume it must be manifests to slurp 83 | data, err := os.ReadFile(path) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | br := &kio.ByteReader{ 89 | Reader: bytes.NewReader(data), 90 | SetAnnotations: map[string]string{ 91 | kioutil.PathAnnotation: path, 92 | }, 93 | } 94 | 95 | // Try to parse the file 96 | nodes, err := br.Read() 97 | if err != nil && filepath.Ext(path) != "" { 98 | return err 99 | } 100 | 101 | // Only keep things that appear to be Kube resources 102 | for _, n := range nodes { 103 | if keepNode(n) { 104 | result = append(result, n) 105 | } 106 | } 107 | } 108 | 109 | return nil 110 | }) 111 | 112 | if err != nil { 113 | return nil, err 114 | } 115 | 116 | return result, nil 117 | } 118 | 119 | // root returns the root path, failing if it cannot be made into an absolute path. 120 | func (r *FileReader) root() (path string, err error) { 121 | path = r.Path 122 | 123 | // If available, use the path resolver 124 | if r.Abs != nil { 125 | path, err = r.Abs(path) 126 | if err != nil { 127 | return "", err 128 | } 129 | } 130 | 131 | // Even if we used Abs, make sure the result is actually absolute 132 | if !filepath.IsAbs(path) { 133 | return "", fmt.Errorf("unable to resolve relative path %s", r.Path) 134 | } 135 | 136 | return path, nil 137 | } 138 | 139 | // readDir attempts to expand a directory into Konjure nodes. If any nodes are returned, traversal of the directory 140 | // is skipped. 141 | func (r *FileReader) readDir(result []*yaml.RNode, path string) ([]*yaml.RNode, error) { 142 | // Read the directory listing, ignore errors 143 | d, err := os.Open(path) 144 | if err != nil { 145 | return nil, nil 146 | } 147 | dirContents, _ := d.Readdirnames(-1) 148 | _ = d.Close() 149 | 150 | // This is a Git repository, but since it is already cloned, we can just skip it 151 | if filepath.Base(path) == ".git" && containsAll(dirContents, "objects", "refs", "HEAD") { // Just sanity check the dir contents 152 | return result, fs.SkipDir 153 | } 154 | 155 | // Look for directory contents that indicate we should handle this specially 156 | for _, name := range dirContents { 157 | switch name { 158 | case "kustomization.yaml", 159 | "kustomization.yml", 160 | "Kustomization": 161 | // The path is a Kustomization root: return a Kustomize resource, so it gets expanded correctly 162 | n, err := konjurev1beta2.GetRNode(&konjurev1beta2.Kustomize{Root: path}) 163 | if err != nil { 164 | return nil, err 165 | } 166 | 167 | result = append(result, n) 168 | return result, fs.SkipDir 169 | 170 | case "Chart.yaml": 171 | // The path is a Helm chart: return a Helm resource, so we don't fail parsing YAML templates 172 | // TODO Read the chart... 173 | return result, fs.SkipDir 174 | } 175 | } 176 | 177 | return result, nil 178 | } 179 | 180 | // keepNode tests the supplied node to see if it should be included in the result. 181 | func keepNode(node *yaml.RNode) bool { 182 | m, err := node.GetMeta() 183 | if err != nil { 184 | return false 185 | } 186 | 187 | // Kind is required 188 | if m.Kind == "" { 189 | return false 190 | } 191 | 192 | switch { 193 | 194 | case m.APIVersion == konjurev1beta2.APIVersion: 195 | // Keep all Konjure resources 196 | return true 197 | 198 | case strings.HasPrefix(m.APIVersion, "kustomize.config.k8s.io/"): 199 | // Keep all Kustomize resources (special case when the Kustomization wasn't expanded) 200 | return true 201 | 202 | case strings.HasSuffix(m.Kind, "List"): 203 | // Keep list resources 204 | return true 205 | 206 | default: 207 | // Keep other resources only if they have a name 208 | return m.Name != "" 209 | } 210 | } 211 | 212 | // containsAll checks that all values are present. 213 | func containsAll(haystack []string, needles ...string) bool { 214 | for i := range haystack { 215 | for j := range needles { 216 | if haystack[i] == needles[j] { 217 | needles = append(needles[:j], needles[j+1:]...) 218 | break 219 | } 220 | } 221 | } 222 | return len(needles) == 0 223 | } 224 | -------------------------------------------------------------------------------- /pkg/konjure/resource.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package konjure 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "reflect" 23 | 24 | "github.com/thestormforge/konjure/internal/readers" 25 | "github.com/thestormforge/konjure/internal/spec" 26 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 27 | "sigs.k8s.io/kustomize/kyaml/kio" 28 | "sigs.k8s.io/kustomize/kyaml/yaml" 29 | ) 30 | 31 | // Resource is a Konjure resource. This type may be used to collect Konjure 32 | // resource specifications: either by URL-like string or by specifying the 33 | // structured form of a resource. Only one of the pointers may be non-nil at 34 | // a time. 35 | type Resource struct { 36 | Resource *konjurev1beta2.Resource `json:"resource,omitempty" yaml:"resource,omitempty"` 37 | Helm *konjurev1beta2.Helm `json:"helm,omitempty" yaml:"helm,omitempty"` 38 | Jsonnet *konjurev1beta2.Jsonnet `json:"jsonnet,omitempty" yaml:"jsonnet,omitempty"` 39 | Kubernetes *konjurev1beta2.Kubernetes `json:"kubernetes,omitempty" yaml:"kubernetes,omitempty"` 40 | Kustomize *konjurev1beta2.Kustomize `json:"kustomize,omitempty" yaml:"kustomize,omitempty"` 41 | Secret *konjurev1beta2.Secret `json:"secret,omitempty" yaml:"secret,omitempty"` 42 | Git *konjurev1beta2.Git `json:"git,omitempty" yaml:"git,omitempty"` 43 | HTTP *konjurev1beta2.HTTP `json:"http,omitempty" yaml:"http,omitempty"` 44 | File *konjurev1beta2.File `json:"file,omitempty" yaml:"file,omitempty"` 45 | 46 | // Some specs (default reader, `data:` URLs, inline resources) resolve to a stream. 47 | raw kio.Reader `json:"-"` // NOTE: when this is non-nil there MUST be a value for `str`! 48 | 49 | // Original string representation this resource was parsed from. 50 | str string `json:"-"` 51 | } 52 | 53 | // NewResource returns a resource for parsing the supplied resource specifications. This 54 | // is a convenience function for abstracting away gratuitous use of the word "resource". 55 | func NewResource(arg ...string) Resource { 56 | r := Resource{Resource: &konjurev1beta2.Resource{Resources: arg}} // Turtles... 57 | if len(arg) == 1 { 58 | r.str = arg[0] 59 | } 60 | return r 61 | } 62 | 63 | // Read returns a KYAML resource nodes representing this Konjure resource. 64 | func (r *Resource) Read() ([]*yaml.RNode, error) { 65 | if r.raw != nil { 66 | return r.raw.Read() 67 | } 68 | 69 | rv := reflect.Indirect(reflect.ValueOf(r)) 70 | for i := 0; i < rv.NumField(); i++ { 71 | if f := rv.Field(i); f.Kind() != reflect.String && !f.IsNil() { 72 | n, err := konjurev1beta2.GetRNode(rv.Field(i).Interface()) 73 | if err != nil { 74 | return nil, err 75 | } 76 | return []*yaml.RNode{n}, nil 77 | } 78 | } 79 | 80 | return nil, fmt.Errorf("resource is missing definition") 81 | } 82 | 83 | // UnmarshalJSON allows a Konjure resource to be represented by a plain URL-like 84 | // string or a typed structure. 85 | func (r *Resource) UnmarshalJSON(bytes []byte) error { 86 | if err := json.Unmarshal(bytes, &r.str); err == nil { 87 | rr, err := (&spec.Parser{}).Decode(r.str) 88 | if err != nil { 89 | return err 90 | } 91 | 92 | if raw, ok := rr.(kio.Reader); ok { 93 | r.raw = raw 94 | return nil 95 | } 96 | 97 | rv := reflect.Indirect(reflect.ValueOf(r)) 98 | rrv := reflect.ValueOf(rr) 99 | for i := 0; i < rv.NumField(); i++ { 100 | if rv.Field(i).Type() == rrv.Type() { 101 | rv.Field(i).Set(rrv) 102 | return nil 103 | } 104 | } 105 | 106 | return fmt.Errorf("unknown resource type: %T", rr) 107 | } 108 | 109 | type rt *Resource 110 | return json.Unmarshal(bytes, rt(r)) 111 | } 112 | 113 | // MarshalJSON produces JSON for this Konjure resource. If it was initially read 114 | // from a string, the string representation is used, however changes made to the 115 | // object will not be reflected. 116 | func (r *Resource) MarshalJSON() ([]byte, error) { 117 | if r.str != "" { 118 | return json.Marshal(r.str) 119 | } 120 | 121 | rv := reflect.Indirect(reflect.ValueOf(r)) 122 | for i := 0; i < rv.NumField(); i++ { 123 | if !rv.Field(i).IsNil() { 124 | str, err := (&spec.Formatter{}).Encode(rv.Field(i).Interface()) 125 | if err != nil { 126 | break 127 | } 128 | return json.Marshal(str) 129 | } 130 | } 131 | 132 | type rt Resource 133 | return json.Marshal((*rt)(r)) 134 | } 135 | 136 | // DeepCopyInto is meant to allow Resource to be used with code generated by 137 | // Kubernetes controller-gen. It is not a true deep copy, it will only copy the 138 | // destination of the non-nil pointers. 139 | func (r *Resource) DeepCopyInto(rout *Resource) { 140 | rout.str = r.str 141 | 142 | rvin := reflect.Indirect(reflect.ValueOf(r)) 143 | rvout := reflect.Indirect(reflect.ValueOf(rout)) 144 | for i := 0; i < rvin.NumField(); i++ { 145 | if f := rvin.Field(i); f.Kind() != reflect.String && !f.IsNil() { 146 | rvout.Field(i).Set(reflect.New(f.Elem().Type())) 147 | rvout.Field(i).Elem().Set(f.Elem()) 148 | } 149 | } 150 | } 151 | 152 | // Resources is a list of Konjure resources, it can be used as a KYAML Reader 153 | // to obtain YAML representations of the Konjure resources. 154 | type Resources []Resource 155 | 156 | var _ kio.Reader = Resources{} 157 | 158 | // Read returns the RNode representations of the Konjure resources. 159 | func (rs Resources) Read() ([]*yaml.RNode, error) { 160 | result := make([]*yaml.RNode, 0, len(rs)) 161 | for i := range rs { 162 | nodes, err := rs[i].Read() 163 | if err != nil { 164 | return nil, err 165 | } 166 | result = append(result, nodes...) 167 | } 168 | 169 | return result, nil 170 | } 171 | 172 | // NewReader returns a KYAML reader for an individual Konjure resource 173 | // specification. If the object is not recognized, the resulting reader will 174 | // silently generate nothing. 175 | func NewReader(obj any) kio.Reader { 176 | if r := readers.New(obj); r != nil { 177 | return r 178 | } 179 | 180 | // Use an empty RNode slice instead of returning nil 181 | return kio.ResourceNodeSlice{} 182 | } 183 | -------------------------------------------------------------------------------- /internal/readers/secret.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "bufio" 21 | "bytes" 22 | "crypto/rand" 23 | "fmt" 24 | "os" 25 | "path" 26 | "strings" 27 | "unicode" 28 | "unicode/utf8" 29 | 30 | "github.com/google/uuid" 31 | "github.com/oklog/ulid/v2" 32 | "github.com/sethvargo/go-password/password" 33 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 34 | "sigs.k8s.io/kustomize/kyaml/yaml" 35 | ) 36 | 37 | type SecretReader struct { 38 | konjurev1beta2.Secret 39 | } 40 | 41 | func (r *SecretReader) Read() ([]*yaml.RNode, error) { 42 | // Build the basic secret node 43 | n, err := yaml.FromMap(map[string]any{"apiVersion": "v1", "kind": "Secret"}) 44 | if err != nil { 45 | return nil, err 46 | } 47 | if err := n.PipeE(yaml.SetK8sName(r.SecretName)); err != nil { 48 | return nil, err 49 | } 50 | if r.Type != "" { 51 | if err := n.PipeE(yaml.SetField("type", yaml.NewStringRNode(r.Type))); err != nil { 52 | return nil, err 53 | } 54 | } 55 | 56 | // Add all the secret data 57 | if err := n.PipeE(yaml.Tee( 58 | yaml.FilterFunc(r.literals), 59 | yaml.FilterFunc(r.files), 60 | yaml.FilterFunc(r.envs), 61 | yaml.FilterFunc(r.uuids), 62 | yaml.FilterFunc(r.ulids), 63 | yaml.FilterFunc(r.passwords), 64 | )); err != nil { 65 | return nil, err 66 | } 67 | 68 | return []*yaml.RNode{n}, nil 69 | } 70 | 71 | func (r *SecretReader) literals(n *yaml.RNode) (*yaml.RNode, error) { 72 | if len(r.LiteralSources) == 0 { 73 | return n, nil 74 | } 75 | 76 | m := make(map[string]string) 77 | for _, s := range r.LiteralSources { 78 | items := strings.SplitN(s, "=", 2) 79 | if items[0] == "" || len(items) != 2 { 80 | return nil, fmt.Errorf("invalid literal, expected key=value: %s", s) 81 | } 82 | m[items[0]] = strings.Trim(items[1], `"'`) 83 | } 84 | 85 | return n, n.LoadMapIntoSecretData(m) 86 | } 87 | 88 | func (r *SecretReader) files(n *yaml.RNode) (*yaml.RNode, error) { 89 | if len(r.FileSources) == 0 { 90 | return n, nil 91 | } 92 | 93 | m := make(map[string]string) 94 | for _, s := range r.FileSources { 95 | items := strings.SplitN(s, "=", 3) 96 | switch len(items) { 97 | case 1: 98 | data, err := os.ReadFile(items[0]) 99 | if err != nil { 100 | return nil, err 101 | } 102 | m[path.Base(items[0])] = string(data) 103 | 104 | case 2: 105 | if items[0] == "" || items[1] == "" { 106 | return nil, fmt.Errorf("key or file path is missing: %s", s) 107 | } 108 | 109 | data, err := os.ReadFile(items[1]) 110 | if err != nil { 111 | return nil, err 112 | } 113 | m[items[0]] = string(data) 114 | 115 | default: 116 | return nil, fmt.Errorf("key names or file paths cannot contain '='") 117 | } 118 | } 119 | 120 | return n, n.LoadMapIntoSecretData(m) 121 | } 122 | 123 | func (r *SecretReader) envs(n *yaml.RNode) (*yaml.RNode, error) { 124 | if len(r.EnvSources) == 0 { 125 | return n, nil 126 | } 127 | 128 | m := make(map[string]string) 129 | for _, s := range r.EnvSources { 130 | data, err := os.ReadFile(s) 131 | if err != nil { 132 | return nil, err 133 | } 134 | 135 | scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimPrefix(data, []byte{0xEF, 0xBB, 0xBF}))) 136 | currentLine := 0 137 | for scanner.Scan() { 138 | currentLine++ 139 | 140 | line := scanner.Bytes() 141 | if !utf8.Valid(line) { 142 | return nil, fmt.Errorf("line %d has invalid UTF-8 bytes: %s", currentLine, string(line)) 143 | } 144 | 145 | line = bytes.TrimLeftFunc(line, unicode.IsSpace) 146 | if len(line) == 0 || line[0] == '#' { 147 | continue 148 | } 149 | 150 | items := strings.SplitN(string(line), "=", 2) 151 | if len(items) == 2 { 152 | m[items[0]] = items[1] 153 | } else { 154 | m[items[0]] = os.Getenv(items[0]) 155 | } 156 | } 157 | } 158 | 159 | return n, n.LoadMapIntoSecretData(m) 160 | } 161 | 162 | func (r *SecretReader) uuids(n *yaml.RNode) (*yaml.RNode, error) { 163 | if len(r.UUIDSources) == 0 { 164 | return n, nil 165 | } 166 | 167 | m := make(map[string]string) 168 | for _, s := range r.UUIDSources { 169 | v, err := uuid.NewRandom() 170 | if err != nil { 171 | return nil, err 172 | } 173 | m[s] = v.String() 174 | } 175 | 176 | return n, n.LoadMapIntoSecretData(m) 177 | } 178 | 179 | func (r *SecretReader) ulids(n *yaml.RNode) (*yaml.RNode, error) { 180 | if len(r.ULIDSources) == 0 { 181 | return n, nil 182 | } 183 | 184 | m := make(map[string]string) 185 | for _, s := range r.ULIDSources { 186 | v, err := ulid.New(ulid.Now(), rand.Reader) 187 | if err != nil { 188 | return nil, err 189 | } 190 | m[s] = v.String() 191 | } 192 | 193 | return n, n.LoadMapIntoSecretData(m) 194 | } 195 | 196 | func (r *SecretReader) passwords(n *yaml.RNode) (*yaml.RNode, error) { 197 | if len(r.PasswordSources) == 0 { 198 | return n, nil 199 | } 200 | 201 | gen, err := password.NewGenerator(r.PasswordOptions) 202 | if err != nil { 203 | return nil, err 204 | } 205 | 206 | m := make(map[string]string) 207 | for i := range r.PasswordSources { 208 | pwd, err := gen.Generate(passwordArgs(&r.PasswordSources[i])) 209 | if err != nil { 210 | return nil, err 211 | } 212 | m[r.PasswordSources[i].Key] = pwd 213 | } 214 | 215 | return n, n.LoadMapIntoSecretData(m) 216 | } 217 | 218 | func passwordArgs(s *konjurev1beta2.PasswordRecipe) (length int, numDigits int, numSymbols int, noUpper bool, allowRepeat bool) { 219 | if s.Length != nil { 220 | length = *s.Length 221 | } 222 | if s.NumDigits != nil { 223 | numDigits = *s.NumDigits 224 | } 225 | if s.NumSymbols != nil { 226 | numSymbols = *s.NumSymbols 227 | } 228 | if s.NoUpper != nil { 229 | noUpper = *s.NoUpper 230 | } 231 | if s.AllowRepeat != nil { 232 | allowRepeat = *s.AllowRepeat 233 | } 234 | 235 | // TODO Is this reasonable default logic? 236 | if length == 0 { 237 | length = 64 238 | } 239 | if numDigits == 0 && numSymbols+10 <= length { 240 | numDigits = 10 241 | } 242 | if numSymbols == 0 && numDigits+10 <= length { 243 | numSymbols = 10 244 | } 245 | 246 | return length, numDigits, numSymbols, noUpper, allowRepeat 247 | } 248 | -------------------------------------------------------------------------------- /pkg/pipes/cobra.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pipes 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "io" 23 | "os" 24 | "path/filepath" 25 | "strings" 26 | "sync" 27 | 28 | "github.com/mattn/go-isatty" 29 | "github.com/spf13/cobra" 30 | "github.com/thestormforge/konjure/pkg/konjure" 31 | "sigs.k8s.io/kustomize/kyaml/kio" 32 | "sigs.k8s.io/kustomize/kyaml/kio/filters" 33 | "sigs.k8s.io/kustomize/kyaml/kio/kioutil" 34 | "sigs.k8s.io/kustomize/kyaml/yaml" 35 | ) 36 | 37 | // CommandWriterFormatFlag is a global that can be overwritten to change the name 38 | // of the flag used to get the output format from the command. 39 | var CommandWriterFormatFlag = "output" 40 | 41 | // CommandReaders returns KYAML readers for the supplied file name arguments. 42 | func CommandReaders(cmd *cobra.Command, args []string) []kio.Reader { 43 | var inputs []kio.Reader 44 | 45 | var hasDefault bool 46 | for _, filename := range args { 47 | r := &kio.ByteReader{ 48 | Reader: &fileReader{Filename: filename}, 49 | SetAnnotations: map[string]string{kioutil.PathAnnotation: filename}, 50 | } 51 | 52 | // Handle the process relative "default" stream 53 | if filename == "-" { 54 | // Only take stdin once 55 | if hasDefault { 56 | continue 57 | } 58 | hasDefault = true 59 | 60 | r.Reader = cmd.InOrStdin() 61 | 62 | delete(r.SetAnnotations, kioutil.PathAnnotation) 63 | if path, err := filepath.Abs("stdin"); err == nil { 64 | r.SetAnnotations[kioutil.PathAnnotation] = path 65 | } 66 | } 67 | 68 | inputs = append(inputs, r) 69 | } 70 | 71 | return inputs 72 | } 73 | 74 | // CommandWriters returns KYAML writers for the supplied command. 75 | func CommandWriters(cmd *cobra.Command, overwriteFiles bool) []kio.Writer { 76 | var outputs []kio.Writer 77 | 78 | format, _ := cmd.Flags().GetString(CommandWriterFormatFlag) 79 | 80 | if overwriteFiles { 81 | outputs = append(outputs, &overwriteWriter{ 82 | Format: format, 83 | ClearAnnotations: []string{ 84 | kioutil.PathAnnotation, 85 | kioutil.LegacyPathAnnotation, 86 | filters.FmtAnnotation, 87 | }, 88 | }) 89 | } else { 90 | outputs = append(outputs, &konjure.Writer{ 91 | Writer: cmd.OutOrStdout(), 92 | InitialDocumentStart: true, 93 | Format: format, 94 | ClearAnnotations: []string{ 95 | kioutil.PathAnnotation, 96 | kioutil.LegacyPathAnnotation, 97 | filters.FmtAnnotation, 98 | }, 99 | }) 100 | } 101 | 102 | return outputs 103 | } 104 | 105 | // CommandEditor returns a filter which launches the node into an editor for interactive edits. 106 | func CommandEditor(cmd *cobra.Command) yaml.Filter { 107 | return yaml.FilterFunc(func(node *yaml.RNode) (*yaml.RNode, error) { 108 | tmp, err := os.CreateTemp("", strings.ReplaceAll(cmd.CommandPath(), " ", "-")+"-*.yaml") 109 | if err != nil { 110 | return nil, err 111 | } 112 | defer func() { _ = os.Remove(tmp.Name()) }() 113 | 114 | // TODO We should support an option to edit in JSON? 115 | // TODO Should we support the option to force Windows line endings? 116 | err = yaml.NewEncoder(tmp).Encode(node.YNode()) 117 | _ = tmp.Close() 118 | if err != nil { 119 | return nil, err 120 | } 121 | 122 | editor := editorCmd(cmd.Context(), tmp.Name()) 123 | editor.Stdout, editor.Stderr = cmd.OutOrStdout(), cmd.ErrOrStderr() 124 | if f, ok := cmd.InOrStdin().(*os.File); ok && isatty.IsTerminal(f.Fd()) { 125 | editor.Stdin = f 126 | } else if tty, err := os.Open("/dev/tty"); err == nil { 127 | defer tty.Close() 128 | editor.Stdin = tty 129 | } else { 130 | return nil, fmt.Errorf("unable to open terminal") 131 | } 132 | 133 | if err := editor.Run(); err != nil { 134 | return nil, err 135 | } 136 | 137 | result, err := yaml.ReadFile(tmp.Name()) 138 | if errors.Is(err, io.EOF) { 139 | return node, nil 140 | } 141 | return result, err 142 | }) 143 | } 144 | 145 | // fileReader is a reader the lazily opens a file for reading and automatically 146 | // closes it when it hits EOF. 147 | type fileReader struct { 148 | sync.Once 149 | io.ReadCloser 150 | Filename string 151 | } 152 | 153 | // Read performs the lazy open on first read and closes on EOF. 154 | func (r *fileReader) Read(p []byte) (n int, err error) { 155 | r.Once.Do(func() { 156 | r.ReadCloser, err = os.Open(r.Filename) 157 | }) 158 | if err != nil { 159 | return 160 | } 161 | 162 | n, err = r.ReadCloser.Read(p) 163 | if err != io.EOF { 164 | return 165 | } 166 | if closeErr := r.ReadCloser.Close(); closeErr != nil { 167 | err = closeErr 168 | } 169 | return 170 | } 171 | 172 | // overwriteWriter is an alternative to the `kio.LocalPackageWriter` that does 173 | // not make assumptions about "packages" or their shared base directories. 174 | type overwriteWriter struct { 175 | Format string 176 | ClearAnnotations []string 177 | } 178 | 179 | // Write overwrites the supplied nodes back into the files they came from. 180 | func (w *overwriteWriter) Write(nodes []*yaml.RNode) error { 181 | if err := kioutil.DefaultPathAndIndexAnnotation("", nodes); err != nil { 182 | return err 183 | } 184 | 185 | pathIndex := make(map[string][]*yaml.RNode, len(nodes)) 186 | for _, n := range nodes { 187 | if path, err := n.Pipe(yaml.GetAnnotation(kioutil.PathAnnotation)); err == nil { 188 | pathIndex[path.YNode().Value] = append(pathIndex[path.YNode().Value], n) 189 | } 190 | } 191 | for k := range pathIndex { 192 | _ = kioutil.SortNodes(pathIndex[k]) 193 | } 194 | 195 | for k, v := range pathIndex { 196 | if err := w.writeToPath(k, v); err != nil { 197 | return err 198 | } 199 | } 200 | 201 | return nil 202 | } 203 | 204 | // writeToPath writes the supplied nodes into the specified file. 205 | func (w *overwriteWriter) writeToPath(path string, nodes []*yaml.RNode) error { 206 | if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { 207 | return err 208 | } 209 | 210 | f, err := os.Create(path) 211 | if err != nil { 212 | return err 213 | } 214 | defer f.Close() 215 | 216 | return (&konjure.Writer{ 217 | Writer: f, 218 | InitialDocumentStart: true, 219 | Format: w.Format, 220 | ClearAnnotations: w.ClearAnnotations, 221 | }).Write(nodes) 222 | } 223 | -------------------------------------------------------------------------------- /pkg/pipes/helmvalues.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pipes 18 | 19 | import ( 20 | "io/fs" 21 | "os" 22 | 23 | "github.com/thestormforge/konjure/pkg/pipes/internal/strvals" 24 | "sigs.k8s.io/kustomize/kyaml/kio" 25 | "sigs.k8s.io/kustomize/kyaml/yaml" 26 | "sigs.k8s.io/kustomize/kyaml/yaml/merge2" 27 | "sigs.k8s.io/kustomize/kyaml/yaml/walk" 28 | ) 29 | 30 | // HelmValues is a reader that emits resource nodes representing Helm values. 31 | type HelmValues struct { 32 | // User specified values files (via -f/--values). 33 | ValueFiles []string 34 | // User specified values (via --set). 35 | Values []string 36 | // User specified string values (via --set-string). 37 | StringValues []string 38 | // User specified file values (via --set-file). 39 | FileValues []string 40 | 41 | // The file system to use for resolving file contents (defaults to the OS reader). 42 | FS fs.FS 43 | } 44 | 45 | // Empty returns true if there are no values configured on the this Helm values instance. 46 | func (r *HelmValues) Empty() bool { 47 | return len(r.ValueFiles)+len(r.Values)+len(r.StringValues)+len(r.FileValues) == 0 48 | } 49 | 50 | // AsMap converts the configured user specified values into a map of values. 51 | func (r *HelmValues) AsMap() (map[string]any, error) { 52 | base := map[string]any{} 53 | 54 | for _, filePath := range r.ValueFiles { 55 | currentMap := map[string]any{} 56 | 57 | data, err := r.readFile(filePath) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | if err := yaml.Unmarshal([]byte(data), ¤tMap); err != nil { 63 | return nil, err 64 | } 65 | 66 | base = r.MergeMaps(base, currentMap) 67 | } 68 | 69 | for _, value := range r.Values { 70 | if err := strvals.ParseInto(value, base); err != nil { 71 | return nil, err 72 | } 73 | } 74 | 75 | for _, value := range r.StringValues { 76 | if err := strvals.ParseIntoString(value, base); err != nil { 77 | return nil, err 78 | } 79 | } 80 | 81 | for _, value := range r.FileValues { 82 | if err := strvals.ParseIntoFile(value, base, func(rs []rune) (any, error) { return r.readFile(string(rs)) }); err != nil { 83 | return nil, err 84 | } 85 | } 86 | 87 | if len(base) == 0 { 88 | return nil, nil 89 | } 90 | return base, nil 91 | } 92 | 93 | // Read converts the configured user specified values into resource nodes. 94 | func (r *HelmValues) Read() ([]*yaml.RNode, error) { 95 | base, err := r.AsMap() 96 | if err != nil { 97 | return nil, err 98 | } 99 | if len(base) == 0 { 100 | return nil, nil 101 | } 102 | 103 | node := yaml.NewRNode(&yaml.Node{}) 104 | if err := node.YNode().Encode(base); err != nil { 105 | return nil, err 106 | } 107 | return []*yaml.RNode{node}, nil 108 | } 109 | 110 | func (r *HelmValues) readFile(spec string) (string, error) { 111 | // TODO Should we be using something like spec.Parser to pull in data? 112 | 113 | if r.FS != nil { 114 | data, err := fs.ReadFile(r.FS, spec) 115 | return string(data), err 116 | } 117 | 118 | data, err := os.ReadFile(spec) 119 | return string(data), err 120 | } 121 | 122 | // Apply merges these Helm values into the filtered nodes without a schema. 123 | func (r *HelmValues) Apply() yaml.Filter { 124 | return yaml.FilterFunc(func(node *yaml.RNode) (*yaml.RNode, error) { 125 | if r.Empty() { 126 | return node, nil 127 | } 128 | 129 | w := walk.Walker{ 130 | Visitor: merge2.Merger{}, 131 | Sources: []*yaml.RNode{node}, 132 | } 133 | if nodes, err := r.Read(); err != nil { 134 | return nil, err 135 | } else { 136 | w.Sources = append(w.Sources, nodes...) 137 | } 138 | return w.Walk() 139 | }) 140 | } 141 | 142 | // MergeMaps is used to combine results from multiple values. 143 | func (r *HelmValues) MergeMaps(a, b map[string]any) map[string]any { 144 | out := make(map[string]any, len(a)) 145 | for k, v := range a { 146 | out[k] = v 147 | } 148 | for k, v := range b { 149 | if v, ok := v.(map[string]any); ok { 150 | if bv, ok := out[k]; ok { 151 | if bv, ok := bv.(map[string]any); ok { 152 | out[k] = r.MergeMaps(bv, v) 153 | continue 154 | } 155 | } 156 | } 157 | out[k] = v 158 | } 159 | return out 160 | } 161 | 162 | // Flatten returns a filter that merges all the supplied nodes into a single node. 163 | func (r *HelmValues) Flatten() kio.Filter { 164 | return kio.FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 165 | out := make(map[string]any) 166 | for _, n := range nodes { 167 | var m map[string]any 168 | if err := n.YNode().Decode(&m); err != nil { 169 | return nil, err 170 | } 171 | out = r.MergeMaps(out, m) 172 | } 173 | result := yaml.NewMapRNode(nil) 174 | if err := result.YNode().Encode(out); err != nil { 175 | return nil, err 176 | } 177 | return []*yaml.RNode{result}, nil 178 | }) 179 | } 180 | 181 | // Mask returns a filter that either keeps or strips data impacted by these values. 182 | func (r *HelmValues) Mask(keep bool) kio.Filter { 183 | return kio.FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 184 | m, err := r.AsMap() 185 | if err != nil { 186 | return nil, err 187 | } 188 | 189 | result := make([]*yaml.RNode, 0, len(nodes)) 190 | for _, n := range nodes { 191 | if nn, err := mask(n, m, keep); err != nil { 192 | return nil, err 193 | } else if nn != nil { 194 | result = append(result, nn) 195 | } 196 | } 197 | return result, nil 198 | }) 199 | } 200 | 201 | func mask(rn *yaml.RNode, m any, keep bool) (*yaml.RNode, error) { 202 | switch m := m.(type) { 203 | case map[string]any: 204 | if err := yaml.ErrorIfInvalid(rn, yaml.MappingNode); err != nil { 205 | return nil, err 206 | } 207 | 208 | original := rn.Content() 209 | masked := make([]*yaml.Node, 0, len(original)) 210 | for i := 0; i < len(original); i += 2 { 211 | if v, ok := m[original[i].Value]; ok { 212 | // Recursively filter the value 213 | if value, err := mask(yaml.NewRNode(original[i+1]), v, keep); err != nil { 214 | return nil, err 215 | } else if value != nil { 216 | masked = append(masked, original[i], value.YNode()) 217 | } 218 | } else if !keep { 219 | // Just keep it 220 | masked = append(masked, original[i], original[i+1]) 221 | } 222 | } 223 | if len(masked) > 0 { 224 | rn = rn.Copy() 225 | rn.YNode().Content = masked 226 | return rn, nil 227 | } 228 | 229 | default: 230 | if keep && m != nil { 231 | return rn.Copy(), nil 232 | } 233 | } 234 | return nil, nil 235 | } 236 | -------------------------------------------------------------------------------- /internal/readers/jsonnet.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readers 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "io" 23 | "os" 24 | "path/filepath" 25 | "strings" 26 | 27 | "github.com/fatih/color" 28 | "github.com/google/go-jsonnet" 29 | jb "github.com/jsonnet-bundler/jsonnet-bundler/pkg" 30 | "github.com/jsonnet-bundler/jsonnet-bundler/pkg/jsonnetfile" 31 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 32 | "sigs.k8s.io/kustomize/kyaml/kio" 33 | "sigs.k8s.io/kustomize/kyaml/yaml" 34 | ) 35 | 36 | func NewJsonnetReader(js *konjurev1beta2.Jsonnet) kio.Reader { 37 | // Build the reader from the Jsonnet configuration 38 | r := &JsonnetReader{ 39 | JsonnetBundlerPackageHome: js.JsonnetBundlerPackageHome, 40 | FileImporter: jsonnet.FileImporter{ 41 | JPaths: js.JsonnetPath, 42 | }, 43 | MakeVM: func() *jsonnet.VM { 44 | vm := jsonnet.MakeVM() 45 | processParameters(js.ExternalVariables, vm.ExtVar, vm.ExtCode) 46 | processParameters(js.TopLevelArguments, vm.TLAVar, vm.TLACode) 47 | return vm 48 | }, 49 | Filename: js.Filename, 50 | Snippet: js.Code, 51 | } 52 | 53 | // Setup Jsonnet Bundler 54 | if _, err := os.Stat(jsonnetfile.File); err != nil { 55 | r.JsonnetBundlerPackageHome = "" // No 'jsonnetfile.json' 56 | } else { 57 | if r.JsonnetBundlerPackageHome == "" { 58 | r.JsonnetBundlerPackageHome = "vendor" 59 | } 60 | r.FileImporter.JPaths = append(r.FileImporter.JPaths, js.JsonnetBundlerPackageHome) 61 | 62 | if _, err := os.Stat(r.JsonnetBundlerPackageHome); err == nil && !js.JsonnetBundlerRefresh { 63 | r.JsonnetBundlerPackageHome = "" // Vendor exists and should not be refreshed 64 | } 65 | } 66 | 67 | // Finish the path (it must be reversed since the file importer reads from the end) 68 | r.FileImporter.JPaths = append(r.FileImporter.JPaths, filepath.SplitList(os.Getenv("JSONNET_PATH"))...) 69 | for i, j := 0, len(r.FileImporter.JPaths)-1; i < j; i, j = i+1, j-1 { 70 | r.FileImporter.JPaths[i], r.FileImporter.JPaths[j] = r.FileImporter.JPaths[j], r.FileImporter.JPaths[i] 71 | } 72 | 73 | return r 74 | } 75 | 76 | type JsonnetReader struct { 77 | JsonnetBundlerPackageHome string 78 | FileImporter jsonnet.FileImporter 79 | MakeVM func() *jsonnet.VM 80 | Filename string 81 | Snippet string 82 | } 83 | 84 | func (r *JsonnetReader) Read() ([]*yaml.RNode, error) { 85 | // Before we start, make sure the bundler is up-to-date (i.e. download `/vendor/`) 86 | if r.JsonnetBundlerPackageHome != "" { 87 | if err := r.bundlerEnsure(); err != nil { 88 | return nil, err 89 | } 90 | } 91 | 92 | // Get the configured VM factory or use the default 93 | makeVM := r.MakeVM 94 | if makeVM == nil { 95 | makeVM = jsonnet.MakeVM 96 | } 97 | 98 | // Create a new VM configured to use our `Import` function 99 | vm := makeVM() 100 | vm.Importer(r) 101 | 102 | // TODO This is largely implementing legacy Konjure behavior, is it still valid? 103 | var data string 104 | var err error 105 | if r.Filename != "" { 106 | data, err = vm.EvaluateFile(r.Filename) 107 | } else { 108 | filename := "" 109 | if r.Snippet == "" { 110 | filename = "" 111 | } 112 | data, err = vm.EvaluateAnonymousSnippet(filename, r.Snippet) 113 | } 114 | if err != nil { 115 | return nil, err 116 | } 117 | 118 | return r.parseJSON(data) 119 | } 120 | 121 | func (r *JsonnetReader) Import(importedFrom, importedPath string) (jsonnet.Contents, string, error) { 122 | return r.FileImporter.Import(importedFrom, importedPath) 123 | } 124 | 125 | // bundlerEnsure runs the Jsonnet bundler to ensure any dependencies are present. 126 | func (r *JsonnetReader) bundlerEnsure() error { 127 | // Attempt to find and load the Jsonnet Bundler file 128 | jbfile, _, err := r.Import("", jsonnetfile.File) 129 | if err != nil { 130 | return err 131 | } 132 | jsonnetFile, err := jsonnetfile.Unmarshal([]byte(jbfile.String())) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | // Attempt to find and load the Jsonnet Bundler lock file 138 | jblockfile, _, err := r.Import("", jsonnetfile.LockFile) 139 | if err != nil && !os.IsNotExist(err) { 140 | return err 141 | } 142 | lockFile, err := jsonnetfile.Unmarshal([]byte(jblockfile.String())) 143 | if err != nil { 144 | return err 145 | } 146 | 147 | // Create a temporary directory for the bundler to use 148 | if err := os.MkdirAll(filepath.Join(r.JsonnetBundlerPackageHome, ".tmp"), os.ModePerm); err != nil { 149 | return err 150 | } 151 | 152 | // TODO Append JsonnetBundlerPackageHome to the r.FileImporter JPath iff it is not already there 153 | 154 | // Ignore output when ensuring dependencies are updated 155 | color.Output = io.Discard 156 | 157 | _, err = jb.Ensure(jsonnetFile, r.JsonnetBundlerPackageHome, lockFile.Dependencies) 158 | return err 159 | } 160 | 161 | // parseJson takes Jsonnet output and makes it into resource nodes 162 | func (r *JsonnetReader) parseJSON(j string) ([]*yaml.RNode, error) { 163 | t, err := json.NewDecoder(strings.NewReader(j)).Token() 164 | if err != nil { 165 | return nil, err 166 | } 167 | 168 | // Setup a byte reader 169 | br := &kio.ByteReader{SetAnnotations: map[string]string{}} 170 | 171 | // If it looks like a JSON list, trick the parser it by wrapping it with an items field 172 | if t == json.Delim('[') { 173 | br.Reader = strings.NewReader(`{"kind":"List","items":` + j + `}`) 174 | return br.Read() 175 | } 176 | 177 | if t != json.Delim('{') { 178 | return nil, fmt.Errorf("expected JSON object or list") 179 | } 180 | 181 | // Parse the JSON output as YAML 182 | br.Reader = strings.NewReader(j) 183 | result, err := br.Read() 184 | 185 | // Either an error, an empty document, or a `kind: List` that was unwrapped 186 | if err != nil || len(result) != 1 { 187 | return result, err 188 | } 189 | 190 | // The only object in the result is already a resource, just return 191 | if _, err := result[0].GetValidatedMetadata(); err == nil { 192 | return result, nil 193 | } 194 | 195 | // Convert the map of filename->documents into a list where each document has the name as a comment 196 | if err := result[0].VisitFields(func(node *yaml.MapNode) error { 197 | node.Value.YNode().HeadComment = fmt.Sprintf("Source: %s", node.Key.YNode().Value) 198 | result = append(result, node.Value) 199 | return nil 200 | }); err != nil { 201 | return nil, err 202 | } 203 | 204 | return result[1:], nil 205 | } 206 | 207 | // processParameters is a helper to configure the Jsonnet VM. 208 | func processParameters(params []konjurev1beta2.JsonnetParameter, handleVar func(string, string), handleCode func(string, string)) { 209 | for _, p := range params { 210 | if p.String != "" { 211 | handleVar(p.Name, p.String) 212 | } else if p.StringFile != "" { 213 | handleCode(p.Name, fmt.Sprintf("importstr @'%s'", strings.ReplaceAll(p.StringFile, "'", "''"))) 214 | } else if p.Code != "" { 215 | handleCode(p.Name, p.Code) 216 | } else if p.CodeFile != "" { 217 | handleCode(p.Name, fmt.Sprintf("import @'%s'", strings.ReplaceAll(p.StringFile, "'", "''"))) 218 | } 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /pkg/filters/application.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | 7 | "github.com/thestormforge/konjure/internal/application" 8 | "sigs.k8s.io/kustomize/kyaml/yaml" 9 | ) 10 | 11 | // ApplicationFilter produces applications based on the observed resources. 12 | type ApplicationFilter struct { 13 | // Flag indicating if this filter should act as a pass-through. 14 | Enabled bool 15 | // Flag indicating we should show resources which do not belong to an application. 16 | ShowUnownedResources bool 17 | // The ordered names of the labels that contains the application name, default is "app.kubernetes.io/name, k8s-app, app". 18 | ApplicationNameLabels []string 19 | // The name of the label that contains the application instance name, default is "app.kubernetes.io/instance". 20 | ApplicationInstanceLabel string 21 | } 22 | 23 | // Filter keeps all the application resources and creates application resources 24 | // for all other nodes that are not associated with an application. 25 | func (f *ApplicationFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 26 | if !f.Enabled { 27 | return nodes, nil 28 | } 29 | 30 | apps := make(map[yaml.NameMeta]*application.Node) 31 | var err error 32 | var scannedAppLabels bool 33 | 34 | IndexApps: 35 | 36 | // Index the existing applications 37 | nodes, err = application.Index(nodes, apps) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | // Remove all the resources that belong to an existing application 43 | for _, app := range apps { 44 | nodes, err = app.Filter(nodes) 45 | if err != nil { 46 | return nil, err 47 | } 48 | } 49 | 50 | // Try to create applications from resource labels 51 | if !scannedAppLabels { 52 | var appsFromLabels []*yaml.RNode 53 | for _, node := range nodes { 54 | app, err := f.appFromLabels(node) 55 | if err != nil { 56 | return nil, err 57 | } 58 | if app != nil { 59 | appsFromLabels = append(appsFromLabels, app) 60 | } 61 | } 62 | 63 | scannedAppLabels = true 64 | if len(appsFromLabels) > 0 { 65 | nodes = append(nodes, appsFromLabels...) 66 | goto IndexApps 67 | } 68 | } 69 | 70 | // Drop the resources that weren't owned by an application 71 | if !f.ShowUnownedResources { 72 | nodes = nil 73 | } 74 | 75 | // Add the applications to list of remaining nodes 76 | for _, app := range apps { 77 | nodes = append(nodes, app.Node) 78 | } 79 | return nodes, nil 80 | } 81 | 82 | // appFromLabels attempts to use the recommended `app.kubernetes.io/*` labels 83 | // from the supplied resource node to generate a new application node. If an 84 | // application cannot be created, this function will just return nil. 85 | func (f *ApplicationFilter) appFromLabels(n *yaml.RNode) (*yaml.RNode, error) { 86 | md, err := n.GetMeta() 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | nameLabelKeys := f.ApplicationNameLabels 92 | if len(nameLabelKeys) == 0 { 93 | nameLabelKeys = []string{application.LabelName, "k8s-app", "app"} 94 | } 95 | var nameLabelKey, nameLabel string 96 | for i := len(nameLabelKeys) - 1; i >= 0; i-- { 97 | if md.Labels[nameLabelKeys[i]] != "" { 98 | nameLabelKey = nameLabelKeys[i] 99 | nameLabel = md.Labels[nameLabelKey] 100 | } 101 | } 102 | 103 | instanceLabelKey := f.ApplicationInstanceLabel 104 | if instanceLabelKey == "" { 105 | instanceLabelKey = application.LabelInstance 106 | } 107 | 108 | instanceLabel := md.Labels[instanceLabelKey] 109 | partOfLabel := md.Labels[application.LabelPartOf] 110 | versionLabel := md.Labels[application.LabelVersion] 111 | 112 | // Special case: let a Helm chart define a missing name/version 113 | if nameLabel == "" || versionLabel == "" { 114 | if helmChartLabel := md.Labels[application.LabelHelmChart]; helmChartLabel != "" { 115 | chartName, chartVersion := splitHelmChart(helmChartLabel) 116 | if nameLabel == "" { 117 | nameLabel = chartName 118 | } 119 | if versionLabel == "" { 120 | versionLabel = chartVersion 121 | } 122 | } 123 | } 124 | 125 | // Build a pipeline of changes, the order in which we add to this list will 126 | // impact what the final document looks like 127 | var pp []yaml.Filter 128 | 129 | // Add Kubernetes metadata for the application 130 | pp = append(pp, yaml.Tee(yaml.SetField(yaml.APIVersionField, yaml.NewStringRNode("app.k8s.io/v1beta1")))) 131 | pp = append(pp, yaml.Tee(yaml.SetField(yaml.KindField, yaml.NewStringRNode("Application")))) 132 | if md.Namespace != "" { 133 | pp = append(pp, yaml.Tee(yaml.SetK8sNamespace(md.Namespace))) 134 | } 135 | 136 | // We need a name for the application: this is where the Application SIG 137 | // spec is a little confusing. They show examples where the application name 138 | // is something like "wordpress-01" implying that it is actually the 139 | // value of the "instance" label that corresponds to the application name. 140 | switch { 141 | case instanceLabel != "" && nameLabel != "": 142 | pp = append(pp, yaml.Tee(yaml.SetK8sName(instanceLabel))) 143 | pp = append(pp, yaml.Tee(yaml.SetLabel(application.LabelName, nameLabel))) 144 | case instanceLabel != "": 145 | pp = append(pp, yaml.Tee(yaml.SetK8sName(instanceLabel))) 146 | pp = append(pp, yaml.Tee(yaml.SetLabel(application.LabelName, instanceLabel))) 147 | case nameLabel != "": 148 | pp = append(pp, yaml.Tee(yaml.SetK8sName(nameLabel))) 149 | pp = append(pp, yaml.Tee(yaml.SetLabel(application.LabelName, nameLabel))) 150 | default: 151 | // With no name we cannot create an application 152 | return nil, nil 153 | } 154 | 155 | // Add match labels for the current resource 156 | if nameLabel != "" { 157 | pp = append(pp, yaml.Tee( 158 | yaml.LookupCreate(yaml.MappingNode, "spec", "selector", "matchLabels"), 159 | yaml.Tee(yaml.SetField(nameLabelKey, yaml.NewStringRNode(nameLabel))), 160 | )) 161 | } 162 | if instanceLabel != "" { 163 | pp = append(pp, yaml.Tee( 164 | yaml.LookupCreate(yaml.MappingNode, "spec", "selector", "matchLabels"), 165 | yaml.Tee(yaml.SetField(instanceLabelKey, yaml.NewStringRNode(instanceLabel))), 166 | )) 167 | } 168 | 169 | // Add the current resource type as one of the component kinds supported by the application 170 | pp = append(pp, yaml.Tee( 171 | yaml.LookupCreate(yaml.SequenceNode, "spec", "componentKinds"), 172 | yaml.Append(&yaml.Node{Kind: yaml.MappingNode}), 173 | yaml.Tee(yaml.SetField("group", yaml.NewStringRNode(application.StripVersion(md.APIVersion)))), 174 | yaml.Tee(yaml.SetField("kind", yaml.NewStringRNode(md.Kind))), 175 | )) 176 | 177 | // Consider part-of the application type, falling back on the name label 178 | // IFF we are using the instance label as the app name... 179 | appType := partOfLabel 180 | if appType == "" && instanceLabel != "" { 181 | appType = nameLabel 182 | } 183 | if appType != "" { 184 | pp = append(pp, yaml.Tee( 185 | yaml.LookupCreate(yaml.ScalarNode, "spec", "descriptor", "type"), 186 | yaml.Tee(yaml.Set(yaml.NewStringRNode(appType))), 187 | )) 188 | } 189 | 190 | // Add the application version 191 | if versionLabel != "" { 192 | pp = append(pp, yaml.Tee( 193 | yaml.LookupCreate(yaml.ScalarNode, "spec", "descriptor", "version"), 194 | yaml.Tee(yaml.Set(yaml.NewStringRNode(versionLabel))), 195 | )) 196 | } 197 | 198 | // Run the constructed pipeline over a new document 199 | return yaml.NewRNode(&yaml.Node{ 200 | Kind: yaml.DocumentNode, 201 | Content: []*yaml.Node{{Kind: yaml.MappingNode}}, 202 | }).Pipe(pp...) 203 | } 204 | 205 | // splitHelmChart splits a Helm chart into it's name and version. 206 | func splitHelmChart(chart string) (name string, version string) { 207 | name = regexp.MustCompile(`-(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)`).Split(chart, 2)[0] 208 | version = strings.TrimPrefix(strings.TrimPrefix(chart, name), "-") 209 | return 210 | } 211 | -------------------------------------------------------------------------------- /internal/spec/parser_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package spec 18 | 19 | import ( 20 | "bytes" 21 | "strings" 22 | "testing" 23 | 24 | "github.com/stretchr/testify/assert" 25 | konjurev1beta2 "github.com/thestormforge/konjure/pkg/api/core/v1beta2" 26 | "sigs.k8s.io/kustomize/kyaml/kio" 27 | ) 28 | 29 | func TestParser_Decode(t *testing.T) { 30 | cases := []struct { 31 | desc string 32 | parser Parser 33 | spec string 34 | expected any 35 | }{ 36 | { 37 | desc: "default reader", 38 | spec: "-", 39 | parser: Parser{Reader: strings.NewReader("test")}, 40 | expected: &kio.ByteReader{Reader: strings.NewReader("test")}, 41 | }, 42 | { 43 | desc: "postgres example", 44 | spec: "github.com/thestormforge/examples/postgres/application", 45 | expected: &konjurev1beta2.Git{ 46 | Repository: "https://github.com/thestormforge/examples.git", 47 | Context: "postgres/application", 48 | }, 49 | }, 50 | { 51 | desc: "kubernetes default deployments of application 'test'", 52 | spec: "k8s:default/deployments?labelSelector=app.kubernetes.io/name%3Dtest", 53 | expected: &konjurev1beta2.Kubernetes{ 54 | Namespace: "default", 55 | Types: []string{"deployments"}, 56 | Selector: "app.kubernetes.io/name=test", 57 | }, 58 | }, 59 | { 60 | desc: "kubernetes default namespace", 61 | spec: "k8s:default", 62 | expected: &konjurev1beta2.Kubernetes{ 63 | Namespace: "default", 64 | }, 65 | }, 66 | { 67 | desc: "kubernetes all deployments", 68 | spec: "k8s:/deployments", 69 | expected: &konjurev1beta2.Kubernetes{ 70 | Types: []string{"deployments"}, 71 | }, 72 | }, 73 | { 74 | desc: "file plain URI", 75 | spec: "file:/foo/bar/", 76 | expected: &konjurev1beta2.File{ 77 | Path: "/foo/bar", 78 | }, 79 | }, 80 | { 81 | desc: "file host URI", 82 | spec: "file://localhost/foo/bar", 83 | expected: &konjurev1beta2.File{ 84 | Path: "/foo/bar", 85 | }, 86 | }, 87 | { 88 | desc: "data URI", 89 | spec: "data:,Hello%2C%20World!", 90 | expected: &kio.ByteReader{ 91 | Reader: bytes.NewReader([]byte("Hello, World!")), 92 | }, 93 | }, 94 | { 95 | desc: "data URI base64", 96 | spec: "data:;base64,SGVsbG8sIFdvcmxkIQ==", 97 | expected: &kio.ByteReader{ 98 | Reader: bytes.NewReader([]byte("Hello, World!")), 99 | }, 100 | }, 101 | { 102 | desc: "helm repo without path", 103 | parser: Parser{HelmRepositoryConfig: HelmRepositoryConfig{Repositories: []HelmRepository{ 104 | {Name: "stable", URL: "https://kubernetes-charts.storage.googleapis.com"}, 105 | }}}, 106 | spec: "helm://stable/elasticsearch", 107 | expected: &konjurev1beta2.Helm{ 108 | Chart: "elasticsearch", 109 | Repository: "https://kubernetes-charts.storage.googleapis.com", 110 | }, 111 | }, 112 | { 113 | desc: "helm repo with path", 114 | parser: Parser{HelmRepositoryConfig: HelmRepositoryConfig{Repositories: []HelmRepository{ 115 | {Name: "bitnami", URL: "https://charts.bitnami.com/bitnami"}, 116 | }}}, 117 | spec: "helm://bitnami/nginx", 118 | expected: &konjurev1beta2.Helm{ 119 | Chart: "nginx", 120 | Repository: "https://charts.bitnami.com/bitnami", 121 | }, 122 | }, 123 | { 124 | desc: "helm download link", 125 | spec: "helm::https://charts.bitnami.com/bitnami/nginx-8.7.1.tgz", 126 | expected: &konjurev1beta2.Helm{ 127 | Chart: "nginx", 128 | Version: "8.7.1", 129 | Repository: "https://charts.bitnami.com/bitnami", 130 | }, 131 | }, 132 | 133 | // URLs to web blobs shouldn't require a full clone 134 | { 135 | desc: "GitHub web blob", 136 | spec: "https://github.com/someorg/somerepo/blob/master/somedir/somefile.yaml", 137 | expected: &konjurev1beta2.HTTP{ 138 | URL: "https://raw.githubusercontent.com/someorg/somerepo/master/somedir/somefile.yaml", 139 | }, 140 | }, 141 | 142 | // Web trees do 143 | { 144 | desc: "GitHub web blob tree", 145 | spec: "https://github.com/someorg/somerepo/tree/master/somedir", 146 | expected: &konjurev1beta2.Git{ 147 | Repository: "https://github.com/someorg/somerepo.git", 148 | Context: "somedir", 149 | Refspec: "master", 150 | }, 151 | }, 152 | 153 | // These are a bunch of test cases from Kustomize for Git URLs 154 | { 155 | desc: "kustomize-tc-0", 156 | spec: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir", 157 | expected: &konjurev1beta2.Git{ 158 | Repository: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo", 159 | Context: "somedir", 160 | }, 161 | }, 162 | { 163 | desc: "kustomize-tc-1", 164 | spec: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir?ref=testbranch", 165 | expected: &konjurev1beta2.Git{ 166 | Repository: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo", 167 | Context: "somedir", 168 | Refspec: "testbranch", 169 | }, 170 | }, 171 | { 172 | desc: "kustomize-tc-2", 173 | spec: "https://fabrikops2.visualstudio.com/someorg/somerepo?ref=master", 174 | expected: &konjurev1beta2.Git{ 175 | Repository: "https://fabrikops2.visualstudio.com/someorg/somerepo", 176 | Refspec: "master", 177 | }, 178 | }, 179 | { 180 | desc: "kustomize-tc-3", 181 | spec: "http://github.com/someorg/somerepo/somedir", 182 | expected: &konjurev1beta2.Git{ 183 | Repository: "https://github.com/someorg/somerepo.git", 184 | Context: "somedir", 185 | }, 186 | }, 187 | { 188 | desc: "kustomize-tc-4", 189 | spec: "git@github.com:someorg/somerepo/somedir", 190 | expected: &konjurev1beta2.Git{ 191 | Repository: "git@github.com:someorg/somerepo.git", 192 | Context: "somedir", 193 | }, 194 | }, 195 | // This doesn't seem valid, an SCP-like spec can't have a port number 196 | //{ 197 | // desc: "kustomize-tc-5", 198 | // spec: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?ref=v0.1.0", 199 | // expected: &konjurev1beta2.Git{ 200 | // Repository: url.URL{User: url.User("git"), Host: "gitlab2.sqtools.ru:10022", Path: "/infra/kubernetes/thanos-base.git"}, 201 | // Refspec: "v0.1.0", 202 | // }, 203 | //}, 204 | { 205 | desc: "kustomize-tc-6", 206 | spec: "git@bitbucket.org:company/project.git//path?ref=branch", 207 | expected: &konjurev1beta2.Git{ 208 | Repository: "git@bitbucket.org:company/project.git", 209 | Context: "path", 210 | Refspec: "branch", 211 | }, 212 | }, 213 | { 214 | desc: "kustomize-tc-7", 215 | spec: "https://itfs.mycompany.com/collection/project/_git/somerepos", 216 | expected: &konjurev1beta2.Git{ 217 | Repository: "https://itfs.mycompany.com/collection/project/_git/somerepos", 218 | }, 219 | }, 220 | { 221 | desc: "kustomize-tc-8", 222 | spec: "https://itfs.mycompany.com/collection/project/_git/somerepos?version=v1.0.0", 223 | expected: &konjurev1beta2.Git{ 224 | Repository: "https://itfs.mycompany.com/collection/project/_git/somerepos", 225 | Refspec: "v1.0.0", 226 | }, 227 | }, 228 | { 229 | desc: "kustomize-tc-9", 230 | spec: "https://itfs.mycompany.com/collection/project/_git/somerepos/somedir?version=v1.0.0", 231 | expected: &konjurev1beta2.Git{ 232 | Repository: "https://itfs.mycompany.com/collection/project/_git/somerepos", 233 | Context: "somedir", 234 | Refspec: "v1.0.0", 235 | }, 236 | }, 237 | { 238 | desc: "kustomize-tc-10", 239 | spec: "git::https://itfs.mycompany.com/collection/project/_git/somerepos", 240 | expected: &konjurev1beta2.Git{ 241 | Repository: "https://itfs.mycompany.com/collection/project/_git/somerepos", 242 | }, 243 | }, 244 | } 245 | for _, c := range cases { 246 | t.Run(c.desc, func(t *testing.T) { 247 | actual, err := c.parser.Decode(c.spec) 248 | if assert.NoError(t, err) { 249 | assert.Equal(t, c.expected, actual) 250 | } 251 | }) 252 | } 253 | } 254 | 255 | func TestParseSpecFailures(t *testing.T) { 256 | cases := []struct { 257 | desc string 258 | reader Parser 259 | spec string 260 | errString string 261 | }{ 262 | //{}, 263 | } 264 | for _, c := range cases { 265 | t.Run(c.desc, func(t *testing.T) { 266 | _, err := c.reader.Decode(c.spec) 267 | assert.EqualError(t, err, c.errString) 268 | }) 269 | } 270 | } 271 | -------------------------------------------------------------------------------- /pkg/filters/filters.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package filters 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "k8s.io/kube-openapi/pkg/validation/spec" 24 | "sigs.k8s.io/kustomize/kyaml/kio" 25 | "sigs.k8s.io/kustomize/kyaml/openapi" 26 | "sigs.k8s.io/kustomize/kyaml/yaml" 27 | "sigs.k8s.io/kustomize/kyaml/yaml/merge2" 28 | "sigs.k8s.io/kustomize/kyaml/yaml/walk" 29 | ) 30 | 31 | // FilterOne is the opposite of kio.FilterAll, useful if you have a filter that 32 | // is optimized for filtering batches of nodes, but you just need to call `Pipe` 33 | // on a single node. 34 | func FilterOne(f kio.Filter) yaml.Filter { 35 | return yaml.FilterFunc(func(node *yaml.RNode) (*yaml.RNode, error) { 36 | nodes, err := f.Filter([]*yaml.RNode{node}) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | if len(nodes) == 1 { 42 | return nodes[0], nil 43 | } 44 | 45 | return nil, nil 46 | }) 47 | } 48 | 49 | // FilterAll is similar to `kio.FilterAll` except instead of evaluating for side 50 | // effects, only the non-nil nodes returned by the filter are preserved. 51 | func FilterAll(f yaml.Filter) kio.Filter { 52 | return kio.FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 53 | var result []*yaml.RNode 54 | for i := range nodes { 55 | n, err := f.Filter(nodes[i]) 56 | if err != nil { 57 | return nil, err 58 | } 59 | if yaml.IsMissingOrNull(n) { 60 | continue 61 | } 62 | 63 | result = append(result, n) 64 | } 65 | return result, nil 66 | }) 67 | } 68 | 69 | // Has is similar to `yaml.Tee` except it only produces a result if the supplied 70 | // functions evaluate to a non-nil result. 71 | func Has(functions ...yaml.Filter) yaml.Filter { 72 | return yaml.FilterFunc(func(rn *yaml.RNode) (*yaml.RNode, error) { 73 | n, err := rn.Pipe(functions...) 74 | if err != nil { 75 | return nil, err 76 | } 77 | if yaml.IsMissingOrNull(n) { 78 | return nil, nil 79 | } 80 | 81 | return rn, nil 82 | }) 83 | } 84 | 85 | // When is used for conditionally constructing filter chains. Unlike other filters, the condition 86 | // is evaluated before the pipeline is executed. This is primarily useful to avoid needing to declare 87 | // a pipeline as a slice of filters which is then built up conditionally. 88 | func When(condition bool, filter yaml.Filter) yaml.Filter { 89 | if condition { 90 | return filter 91 | } 92 | return yaml.FilterFunc(func(object *yaml.RNode) (*yaml.RNode, error) { return object, nil }) 93 | } 94 | 95 | // TeeMatched acts as a "tee" filter for nodes matched by the supplied path matcher: 96 | // each matched node is processed by the supplied filters and the result of the 97 | // entire operation is the initial node (or an error). 98 | func TeeMatched(pathMatcher yaml.PathMatcher, filters ...yaml.Filter) TeeMatchedFilter { 99 | return TeeMatchedFilter{ 100 | PathMatcher: pathMatcher, 101 | Filters: filters, 102 | } 103 | } 104 | 105 | // TeeMatchedFilter is a filter that applies a set of filters to the nodes 106 | // matched by a path matcher. 107 | type TeeMatchedFilter struct { 108 | PathMatcher yaml.PathMatcher 109 | Filters []yaml.Filter 110 | } 111 | 112 | // Filter always returns the supplied node, however all matching nodes will have 113 | // been processed by the configured filters. 114 | func (f TeeMatchedFilter) Filter(rn *yaml.RNode) (*yaml.RNode, error) { 115 | matches, err := f.PathMatcher.Filter(rn) 116 | if err != nil { 117 | return nil, err 118 | } 119 | if err := matches.VisitElements(f.visitMatched); err != nil { 120 | return nil, err 121 | } 122 | return rn, nil 123 | } 124 | 125 | // visitMatched is used internally to preserve the field path and apply the 126 | // configured filters. 127 | func (f TeeMatchedFilter) visitMatched(node *yaml.RNode) error { 128 | matches := f.PathMatcher.Matches[node.YNode()] 129 | matchIndex := len(matches) 130 | for _, p := range f.PathMatcher.Path { 131 | if yaml.IsListIndex(p) && matchIndex > 0 { 132 | matchIndex-- 133 | name, _, _ := yaml.SplitIndexNameValue(p) 134 | p = fmt.Sprintf("[%s=%s]", name, matches[matchIndex]) 135 | } 136 | node.AppendToFieldPath(p) 137 | } 138 | 139 | return node.PipeE(f.Filters...) 140 | } 141 | 142 | // Flatten never returns more than a single node, every other node is merged 143 | // into that first node using the supplied schema 144 | func Flatten(schema *spec.Schema) kio.Filter { 145 | return kio.FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) { 146 | w := walk.Walker{Visitor: merge2.Merger{}} 147 | if schema != nil { 148 | w.Schema = &openapi.ResourceSchema{Schema: schema} 149 | } 150 | 151 | for i := len(nodes); i > 1; i-- { 152 | w.Sources = nodes[i-2 : i] 153 | if _, err := w.Walk(); err != nil { 154 | return nil, err 155 | } 156 | nodes = nodes[:i-1] 157 | } 158 | return nodes, nil 159 | }) 160 | } 161 | 162 | // Pipeline is an alternate to the kio.Pipeline. This pipeline has the following differences: 163 | // 1. The read/filter is separated so this pipeline can be used as a reader in another pipeline 164 | // 2. This pipeline does not try to reconcile Kustomize annotations 165 | // 3. This pipeline does not support callbacks 166 | // 4. This pipeline implicitly clears empty annotations 167 | type Pipeline struct { 168 | Inputs []kio.Reader 169 | Filters []kio.Filter 170 | Outputs []kio.Writer 171 | ContinueOnEmptyResult bool 172 | } 173 | 174 | // Read evaluates the inputs and filters, ignoring the writers. 175 | func (p *Pipeline) Read() ([]*yaml.RNode, error) { 176 | var result []*yaml.RNode 177 | 178 | // Read the inputs 179 | for _, input := range p.Inputs { 180 | nodes, err := input.Read() 181 | if err != nil { 182 | return nil, err 183 | } 184 | result = append(result, nodes...) 185 | } 186 | 187 | // Apply the filters 188 | for _, filter := range p.Filters { 189 | var err error 190 | result, err = filter.Filter(result) 191 | if err != nil { 192 | return nil, err 193 | } 194 | 195 | // Allow the filter loop to be stopped early if it goes empty 196 | if len(result) == 0 && !p.ContinueOnEmptyResult { 197 | break 198 | } 199 | } 200 | 201 | // Clear empty annotations on all nodes in the result 202 | for _, node := range result { 203 | if err := yaml.ClearEmptyAnnotations(node); err != nil { 204 | return nil, err 205 | } 206 | } 207 | 208 | return result, nil 209 | } 210 | 211 | // Execute reads and filters the nodes before sending them to the writers. 212 | func (p *Pipeline) Execute() error { 213 | // Call Read to evaluate the Inputs and Filters 214 | nodes, err := p.Read() 215 | if err != nil { 216 | return err 217 | } 218 | 219 | // Check to see if the writers support empty node lists 220 | if len(nodes) == 0 && !p.ContinueOnEmptyResult { 221 | return nil 222 | } 223 | 224 | for _, output := range p.Outputs { 225 | if err := output.Write(nodes); err != nil { 226 | return err 227 | } 228 | } 229 | return nil 230 | } 231 | 232 | // PipeOne acts as a pipeline for a single node. If the supplied reader produces more than one node, this will fail. 233 | func PipeOne(r kio.Reader, f ...yaml.Filter) (*yaml.RNode, error) { 234 | nodes, err := r.Read() 235 | if err != nil || len(nodes) == 0 { 236 | return nil, err 237 | } 238 | if len(nodes) == 1 { 239 | return nodes[0].Pipe(f...) 240 | } 241 | return nil, fmt.Errorf("expected a single document, got %d", len(nodes)) 242 | } 243 | 244 | // ContextFilterFunc is a context-aware YAML filter function. 245 | type ContextFilterFunc func(context.Context, *yaml.RNode) (*yaml.RNode, error) 246 | 247 | // WithContext binds a context to a context filter function. 248 | func WithContext(ctx context.Context, f ContextFilterFunc) yaml.Filter { 249 | return yaml.FilterFunc(func(node *yaml.RNode) (*yaml.RNode, error) { 250 | // Check the context error first, mainly for when this is wrapped in FilterAll 251 | if err := ctx.Err(); err != nil { 252 | return nil, err 253 | } 254 | return f(ctx, node) 255 | }) 256 | } 257 | 258 | // ResetStyle clears the style from the YAML nodes. This is one way to make parsing JSON into more useful YAML. 259 | func ResetStyle() yaml.Filter { 260 | var resetStyle func(*yaml.Node) 261 | resetStyle = func(node *yaml.Node) { 262 | node.Style = 0 263 | for _, node := range node.Content { 264 | resetStyle(node) 265 | } 266 | } 267 | return yaml.FilterFunc(func(node *yaml.RNode) (*yaml.RNode, error) { 268 | resetStyle(node.YNode()) 269 | return node, nil 270 | }) 271 | } 272 | -------------------------------------------------------------------------------- /pkg/api/core/v1beta2/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 GramLabs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta2 18 | 19 | import ( 20 | "github.com/sethvargo/go-password/password" 21 | ) 22 | 23 | // Resource is used to expand a list of URL-like specifications into other Konjure resources. 24 | type Resource struct { 25 | // The list of URL-like specifications to convert into Konjure resources. 26 | Resources []string `json:"resources" yaml:"resources"` 27 | } 28 | 29 | // HelmValue specifies a value or value file for configuring a Helm chart. 30 | type HelmValue struct { 31 | // Path to a values.yaml file. 32 | File string `json:"file,omitempty" yaml:"file,omitempty"` 33 | // Name of an individual name/value to set. 34 | Name string `json:"name,omitempty" yaml:"name,omitempty"` 35 | // Value of an individual name/value to set. 36 | Value string `json:"value,omitempty" yaml:"value,omitempty"` 37 | // Flag indicating that numeric like value should be quoted as strings (e.g. for environment variables). 38 | ForceString bool `json:"forceString,omitempty" yaml:"forceString,omitempty"` // TODO Eliminate and use IntOrString? 39 | // Treat value as a file and load the contents in place of the actual value. 40 | LoadFile bool `json:"loadFile,omitempty" yaml:"loadFile,omitempty"` 41 | } 42 | 43 | // Helm is used to expand a Helm chart locally (using `helm template`). 44 | type Helm struct { 45 | // The release name to use when rendering the chart templates. 46 | ReleaseName string `json:"releaseName,omitempty" yaml:"releaseName,omitempty"` 47 | // The namespace to use when rendering the chart templates (this is particularly important for charts that 48 | // may produce resources in multiple namespaces). 49 | ReleaseNamespace string `json:"releaseNamespace,omitempty" yaml:"releaseNamespace,omitempty"` 50 | // The chart name to inflate. 51 | Chart string `json:"chart" yaml:"chart"` 52 | // The specific version of the chart to use (defaults to the latest release). 53 | Version string `json:"version,omitempty" yaml:"version,omitempty"` 54 | // The repository URL to get the chart from. 55 | Repository string `json:"repo" yaml:"repo"` 56 | // The values used to configure the chart. 57 | Values []HelmValue `json:"values,omitempty" yaml:"values,omitempty"` 58 | // Flag to filter out tests from the results. 59 | IncludeTests bool `json:"includeTests,omitempty" yaml:"includeTests,omitempty"` 60 | } 61 | 62 | // JsonnetParameter specifies inputs to a Jsonnet program. 63 | type JsonnetParameter struct { 64 | // The name of the parameter. 65 | Name string `json:"name,omitempty" yaml:"name,omitempty"` 66 | // The string value of the parameter. 67 | String string `json:"string,omitempty" yaml:"string,omitempty"` 68 | // The file name containing a string parameter value. 69 | StringFile string `json:"stringFile,omitempty" yaml:"stringFile,omitempty"` 70 | // Code to include. 71 | Code string `json:"code,omitempty" yaml:"code,omitempty"` 72 | // The file name containing code to include. 73 | CodeFile string `json:"codeFile,omitempty" yaml:"codeFile,omitempty"` 74 | } 75 | 76 | // Jsonnet is used to expand programmatically constructed resources. 77 | type Jsonnet struct { 78 | // The Jsonnet file to evaluate. 79 | Filename string `json:"filename,omitempty" yaml:"filename,omitempty"` 80 | // An anonymous code snippet to evaluate. 81 | Code string `json:"exec,omitempty" yaml:"exec,omitempty"` 82 | // Additional directories to consider when importing additional Jsonnet code. 83 | JsonnetPath []string `json:"jpath,omitempty" yaml:"jpath,omitempty"` 84 | // The list of external variables to evaluate against. 85 | ExternalVariables []JsonnetParameter `json:"extVar,omitempty" yaml:"extVar,omitempty"` 86 | // The list of top level arguments to evaluate against. 87 | TopLevelArguments []JsonnetParameter `json:"topLevelArg,omitempty" yaml:"topLevelArg,omitempty"` 88 | 89 | // Explicit directory to use fo Jsonnet Bundler support (defaults to "vendor" if "jsonnetfile.json" is present). 90 | JsonnetBundlerPackageHome string `json:"jbPkgHome,omitempty" yaml:"jbPkgHome,omitempty"` 91 | // Flag to force a Bundler refresh, even if the package home directory is already present. 92 | JsonnetBundlerRefresh bool `json:"jbRefresh,omitempty" yaml:"jbRefresh,omitempty"` 93 | } 94 | 95 | // Kubernetes is used to expand resources found in a Kubernetes cluster. 96 | type Kubernetes struct { 97 | // The namespace to look for resources in. 98 | Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` 99 | // An explicit list of namespaces to look for resources in. 100 | Namespaces []string `json:"namespaces,omitempty" yaml:"namespaces,omitempty"` 101 | // A label selector matching namespaces to look for resources in. 102 | NamespaceSelector string `json:"namespaceSelector,omitempty" yaml:"namespaceSelector,omitempty"` 103 | // True to consider all namespaces, ignoring the other configuration. 104 | AllNamespaces bool `json:"allNamespaces,omitempty" yaml:"allNamespaces,omitempty"` 105 | // The list of resource types to include. Defaults to "deployments,statefulsets,configmaps". 106 | Types []string `json:"types,omitempty" yaml:"types,omitempty"` 107 | // A label selector to limit which resources are included. Defaults to "" (match everything). 108 | Selector string `json:"selector,omitempty" yaml:"selector,omitempty"` 109 | // A field selector to limit which resources are included. Defaults to "" (match everything). 110 | FieldSelector string `json:"fieldSelector,omitempty" yaml:"fieldSelector,omitempty"` 111 | } 112 | 113 | // Kustomize is used to expand kustomizations. 114 | type Kustomize struct { 115 | // The Kustomize root to build. 116 | Root string `json:"root" yaml:"root"` 117 | } 118 | 119 | // PasswordRecipe is used to configure random password strings for secrets. 120 | type PasswordRecipe struct { 121 | // The key in the secret data field to use. 122 | Key string `json:"key" yaml:"key"` 123 | // The length of the password. 124 | Length *int `json:"length,omitempty" yaml:"length,omitempty"` 125 | // The number of digits to include in the password. 126 | NumDigits *int `json:"numDigits,omitempty" yaml:"numDigits,omitempty"` 127 | // The number of symbol characters to include in the password. 128 | NumSymbols *int `json:"numSymbols,omitempty" yaml:"numSymbols,omitempty"` 129 | // Flag restricting the use of uppercase characters. 130 | NoUpper *bool `json:"noUpper,omitempty" yaml:"noUpper,omitempty"` 131 | // Flag restricting repeating characters. 132 | AllowRepeat *bool `json:"allowRepeat,omitempty" yaml:"allowRepeat,omitempty"` 133 | } 134 | 135 | // Secret is used to expand a Secret resource. 136 | type Secret struct { 137 | // The name of the secret to generate. 138 | SecretName string `json:"secretName" yaml:"secretName"` 139 | // The type of secret to generate. 140 | Type string `json:"type,omitempty" yaml:"type,omitempty"` 141 | 142 | // A list of `key=value` pairs to include on the secret. 143 | LiteralSources []string `json:"literals,omitempty" yaml:"literals,omitempty"` 144 | // A list of files (or `key=filename` pairs) to include on the secret. 145 | FileSources []string `json:"files,omitempty" yaml:"files,omitempty"` 146 | // A list of .env files (files containing `key=value` pairs) to include on the secret. 147 | EnvSources []string `json:"envs,omitempty" yaml:"envs,omitempty"` 148 | // A list of keys to include randomly generated UUIDs for on the secret. 149 | UUIDSources []string `json:"uuids,omitempty" yaml:"uuids,omitempty"` 150 | // A list of keys to include randomly generated ULIDs for on the secret. 151 | ULIDSources []string `json:"ulids,omitempty" yaml:"ulids,omitempty"` 152 | // A list of password recipes to include random strings on the secret. 153 | PasswordSources []PasswordRecipe `json:"passwords,omitempty" yaml:"passwords,omitempty"` 154 | 155 | // Additional configuration for generating passwords. 156 | PasswordOptions *password.GeneratorInput `json:"-" yaml:"-"` 157 | } 158 | 159 | // Git is used to expand full or partial Git repositories. 160 | type Git struct { 161 | // The Git repository URL. 162 | Repository string `json:"repo,omitempty" yaml:"repo,omitempty"` 163 | // The refspec in the repository to checkout. 164 | Refspec string `json:"refspec,omitempty" yaml:"refspec,omitempty"` 165 | // The subdirectory context to limit the Git repository to. 166 | Context string `json:"context,omitempty" yaml:"context,omitempty"` 167 | } 168 | 169 | // HTTP is used to expand HTTP resources. 170 | type HTTP struct { 171 | // The HTTP(S) URL to fetch. 172 | URL string `json:"url" yaml:"url"` 173 | } 174 | 175 | // File is used to expand local file system resources. 176 | type File struct { 177 | // The file (or directory) name to read. 178 | Path string `json:"path" yaml:"path"` 179 | } 180 | --------------------------------------------------------------------------------