├── .github └── workflows │ ├── release.yaml │ └── validate.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── config │ ├── config.go │ └── vars │ │ ├── delete.go │ │ ├── set.go │ │ ├── set_test.go │ │ ├── vars.go │ │ ├── vars_suite_test.go │ │ └── vars_test.go ├── create.go ├── delete.go ├── list.go ├── package │ ├── download.go │ ├── info.go │ ├── login.go │ ├── package.go │ ├── publish.go │ ├── template.go │ └── validate.go ├── root.go └── vars.go ├── docs └── packages │ └── tutorial.md ├── examples ├── complex-types │ ├── manifest.yaml │ ├── overlay │ │ └── app │ │ │ └── complex.sh │ └── terraform │ │ └── module │ │ ├── corral.tf │ │ └── main.tf ├── multistage │ ├── manifest.yaml │ ├── overlay │ │ └── app │ │ │ └── setvar1.sh │ └── terraform │ │ ├── first │ │ ├── corral.tf │ │ └── main.tf │ │ └── second │ │ ├── corral.tf │ │ └── main.tf ├── overlay-filter │ ├── manifest.yaml │ ├── overlay │ │ ├── a │ │ │ └── app │ │ │ │ └── a │ │ ├── b │ │ │ └── app │ │ │ │ └── b │ │ └── c │ │ │ └── app │ │ │ └── c │ └── terraform │ │ └── module │ │ ├── corral.tf │ │ └── main.tf ├── registry │ ├── manifest.yaml │ ├── overlay │ │ ├── etc │ │ │ ├── docker │ │ │ │ └── registry │ │ │ │ │ ├── config.yml │ │ │ │ │ └── ssl │ │ │ │ │ └── .gitkeep │ │ │ └── systemd │ │ │ │ └── system │ │ │ │ └── registry.service │ │ └── usr │ │ │ └── local │ │ │ └── bin │ │ │ └── registry │ └── terraform │ │ └── main │ │ ├── corral.tf │ │ ├── main.tf │ │ └── outputs.tf ├── simple │ ├── manifest.yaml │ ├── overlay │ │ └── app │ │ │ └── setvar1.sh │ └── terraform │ │ └── module │ │ ├── corral.tf │ │ └── main.tf └── template │ ├── README.md │ ├── config.yaml │ ├── simple-test │ ├── manifest.yaml │ ├── overlay │ │ └── app │ │ │ ├── setvar1.sh │ │ │ └── setvar2.sh │ └── terraform │ │ └── simple │ │ └── module │ │ ├── corral.tf │ │ └── main.tf │ └── test │ ├── manifest.yaml │ └── overlay │ └── app │ └── setvar2.sh ├── go.mod ├── go.sum ├── magefiles └── magefile.go ├── magetools └── magetools.go ├── main.go ├── pkg ├── cmd │ └── output.go ├── config │ ├── config.go │ ├── install.go │ ├── install_test.go │ ├── paths.go │ └── root_path.go ├── corral │ ├── corral.go │ ├── node.go │ └── status.go ├── package │ ├── fetcher.go │ ├── load.go │ ├── manifest.go │ ├── manifest_test.go │ ├── package-manifest.schema.json │ ├── package.go │ ├── registry.go │ ├── template.go │ ├── tests │ │ ├── bad-schema.yaml │ │ └── valid.yaml │ ├── upload.go │ ├── validate.go │ └── variable.go ├── shell │ ├── registry.go │ ├── shell.go │ └── shell_test.go ├── vars │ ├── vars.go │ ├── vars_test.go │ └── varset.go └── version │ └── version.go └── tests └── integration ├── complex-types ├── corral_simple_output_test.go └── testdata │ ├── manifest.yaml │ ├── overlay │ └── app │ │ └── setvariables.sh │ └── terraform │ ├── module │ ├── corral.tf │ └── main.tf │ └── variables │ ├── corral.tf │ └── main.tf ├── template ├── template_suite_test.go ├── template_test.go ├── test │ ├── manifest.yaml │ └── overlay │ │ └── app │ │ ├── template1.sh │ │ └── template2.sh └── testdata │ └── template │ ├── config.yaml │ ├── template1 │ ├── manifest.yaml │ └── overlay │ │ └── app │ │ └── template1.sh │ └── template2 │ ├── manifest.yaml │ └── overlay │ └── app │ └── template2.sh └── validate ├── testdata ├── no_module │ └── manifest.yaml ├── no_overlay │ └── manifest.yaml └── valid │ ├── manifest.yaml │ ├── overlay │ └── test.sh │ └── terraform │ └── module │ └── main.tf ├── validate_suite_test.go └── validate_test.go /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | jobs: 9 | release: 10 | name: release 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: actions/setup-go@v2 15 | with: 16 | go-version: "1.20.4" 17 | - name: Run Mage build linux amd64 18 | uses: magefile/mage-action@v2 19 | env: 20 | GOOS: linux 21 | GOARCH: amd64 22 | with: 23 | version: latest 24 | args: build 25 | - name: Run Mage build linux arm64 26 | uses: magefile/mage-action@v2 27 | env: 28 | GOOS: linux 29 | GOARCH: arm64 30 | with: 31 | version: latest 32 | args: build 33 | - name: Run Mage build darwin amd64 34 | uses: magefile/mage-action@v2 35 | env: 36 | GOOS: darwin 37 | GOARCH: amd64 38 | with: 39 | version: latest 40 | args: build 41 | - name: Run Mage build darwin arm64 42 | uses: magefile/mage-action@v2 43 | env: 44 | GOOS: darwin 45 | GOARCH: arm64 46 | with: 47 | version: latest 48 | args: build 49 | - name: Run Mage build windows amd64 50 | uses: magefile/mage-action@v2 51 | env: 52 | GOOS: windows 53 | GOARCH: amd64 54 | with: 55 | version: latest 56 | args: build 57 | - uses: softprops/action-gh-release@v1 58 | env: 59 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 60 | with: 61 | files: dist/* 62 | prerelease: contains(github.ref, 'rc') 63 | generate_release_notes: true 64 | -------------------------------------------------------------------------------- /.github/workflows/validate.yaml: -------------------------------------------------------------------------------- 1 | name: validate 2 | 3 | on: pull_request 4 | 5 | jobs: 6 | lint: 7 | name: validate 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - uses: actions/setup-go@v2 12 | with: 13 | go-version: "1.20.4" 14 | - run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest 15 | - name: mage validate 16 | uses: magefile/mage-action@v2 17 | with: 18 | version: latest 19 | args: validate 20 | test: 21 | name: test 22 | runs-on: ubuntu-latest 23 | steps: 24 | - uses: actions/checkout@v2 25 | - uses: actions/setup-go@v2 26 | with: 27 | go-version: "1.20.4" 28 | - name: mage test 29 | uses: magefile/mage-action@v2 30 | with: 31 | version: latest 32 | args: test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .dev 2 | dist 3 | .idea 4 | 5 | ### Go template 6 | # Binaries for programs and plugins 7 | *.exe 8 | *.exe~ 9 | *.dll 10 | *.so 11 | *.dylib 12 | vendor 13 | 14 | # Test binary, built with `go test -c` 15 | *.test 16 | 17 | # Output of the go coverage tool, specifically when used with LiteIDE 18 | *.out 19 | 20 | # Dependency directories (remove the comment below to include it) 21 | # vendor/ 22 | 23 | ### Windows template 24 | # Windows thumbnail cache files 25 | Thumbs.db 26 | Thumbs.db:encryptable 27 | ehthumbs.db 28 | ehthumbs_vista.db 29 | 30 | # Dump file 31 | *.stackdump 32 | 33 | # Folder config file 34 | [Dd]esktop.ini 35 | 36 | # Recycle Bin used on file shares 37 | $RECYCLE.BIN/ 38 | 39 | # Windows Installer files 40 | *.cab 41 | *.msi 42 | *.msix 43 | *.msm 44 | *.msp 45 | 46 | # Windows shortcuts 47 | *.lnk 48 | 49 | ### macOS template 50 | # General 51 | .DS_Store 52 | .AppleDouble 53 | .LSOverride 54 | 55 | # Icon must end with two \r 56 | Icon 57 | 58 | # Thumbnails 59 | ._* 60 | 61 | # Files that might appear in the root of a volume 62 | .DocumentRevisions-V100 63 | .fseventsd 64 | .Spotlight-V100 65 | .TemporaryItems 66 | .Trashes 67 | .VolumeIcon.icns 68 | .com.apple.timemachine.donotpresent 69 | 70 | # Directories potentially created on remote AFP share 71 | .AppleDB 72 | .AppleDesktop 73 | Network Trash Folder 74 | Temporary Items 75 | .apdisk 76 | 77 | corral 78 | !corral/ 79 | staging 80 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG OS=linux 2 | ARG ARCH=amd64 3 | ARG VERSION=dev 4 | 5 | FROM golang:1.20-alpine as mage 6 | ARG OS 7 | ARG ARCH 8 | ARG VERSION 9 | WORKDIR /app 10 | 11 | COPY . . 12 | 13 | ARG TARGET=build 14 | 15 | RUN apk update && \ 16 | apk upgrade && \ 17 | apk add curl git 18 | 19 | RUN curl -sLf https://github.com/magefile/mage/releases/download/v1.13.0/mage_1.13.0_Linux-64bit.tar.gz | tar xvzf - -C /usr/bin && chmod +x /usr/bin/mage 20 | 21 | ENV GOOS=${OS} 22 | ENV GOARCH=${ARCH} 23 | ENV VERSION=${VERSION} 24 | 25 | RUN mage -v ${TARGET} ${VERSION} 26 | 27 | FROM scratch 28 | ARG OS 29 | ARG ARCH 30 | WORKDIR /app 31 | 32 | COPY --from=mage /app/dist/corral-${OS}-${ARCH} /usr/bin/corral 33 | 34 | ENTRYPOINT ["corral"] 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rancherlabs/corral/cab06894ef96a71f7becccee8ebcb2a762a3eeca/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Corral 2 | 3 | [![Go Report Card](https://goreportcard.com/badge/github.com/rancherlabs/corral)](https://goreportcard.com/badge/github.com/rancherlabs/corral) 4 | 5 | Corral is a CLI tool for creating and packaging reproducible development environments. Corral allows developers to manage multiple environments, called corrals, and provision them consistently using shared packages. 6 | 7 | # Installation 8 | 9 | To install corral download the latest binary from the [releases](https://github.com/rancherlabs/corral/releases). 10 | 11 | ## First Time Setup 12 | 13 | Before we can use corral we need to run the first time setup. 14 | 15 | ```shell 16 | corral config 17 | ``` 18 | 19 | We also want to set a few global variables, `digitalocean_token` and `digitalocean_domain`. Any variables set with config will be passed to all corrals. This is useful for setting things like cloud credentials or ssh keys. You can always override these values when creating a new corral with the `-v` flag. 20 | 21 | ```shell 22 | corral config vars set digitalocean_token $MY_DO_TOKEN 23 | corral config vars set digitalocean_domain $MY_DO_DOMAIN 24 | ``` 25 | 26 | ## Create 27 | 28 | First we need to create our corral. 29 | 30 | ```shell 31 | corral create simple ghcr.io/rancherlabs/corral/k3s:latest 32 | ``` 33 | 34 | Once this command finishes we will have a Digitalocean droplet running k3s configured. 35 | 36 | ## List 37 | 38 | We can always check what corrals we have running by listing them. 39 | 40 | ```shell 41 | corral list 42 | ``` 43 | 44 | ## Vars 45 | Next lets connect to our cluster. We can get the Kubernetes configuration file from the corral's variables. The file is base64 encoded, so we will need to decode it before we can use it. 46 | 47 | ```shell 48 | corral vars simple kubeconfig | base64 --decode > simple.yaml 49 | kubectl --kubeconfig simple.yaml get nodes 50 | ``` 51 | 52 | ## Delete 53 | 54 | Once we are done using the cluster we can delete it and clean up all the resources generated in Digitalocean. 55 | 56 | ```shell 57 | corral delete simple 58 | ``` 59 | 60 | # What is a corral? 61 | A corral is a collection of resources in a remote environment. Think of it as a way to track the environments you set up and how you set them up. Corrals are created from packages. 62 | 63 | # What is a corral package? 64 | A corral package is a description of an environment. A package consists of one terraform module and filesystem overlay. The terraform module is used to provision cloud resources. Once the resources are created corral will upload the files in the overlay folder to any nodes generated by terraform and call the commands defined in the manifest. 65 | 66 | # How do I create a package? 67 | [See the tutorial here.](docs/packages/tutorial.md) -------------------------------------------------------------------------------- /cmd/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/user" 7 | "path/filepath" 8 | 9 | "github.com/rancherlabs/corral/cmd/config/vars" 10 | "github.com/rancherlabs/corral/pkg/config" 11 | "github.com/sirupsen/logrus" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | func NewCommandConfig() *cobra.Command { 16 | cmd := &cobra.Command{ 17 | Use: "config", 18 | Short: "Set global configuration and download dependencies.", 19 | Long: "Set global configuration and download dependencies.", 20 | Run: configCorral, 21 | } 22 | 23 | cmd.Flags().String("user_id", "", "The user id is used by packages to help identify resources.") 24 | cmd.Flags().String("public_key", "", "Path to a public key you want packages to install on nodes.") 25 | 26 | cmd.AddCommand(vars.NewVarsCommand()) 27 | 28 | return cmd 29 | } 30 | 31 | func configCorral(cmd *cobra.Command, _ []string) { 32 | cfg, _ := config.Load() 33 | userId, _ := cmd.Flags().GetString("user_id") 34 | if userId != "" { 35 | cfg.UserID = userId 36 | } 37 | userPublicKeyPath, _ := cmd.Flags().GetString("public_key") 38 | if userPublicKeyPath != "" { 39 | cfg.UserPublicKeyPath = userPublicKeyPath 40 | } 41 | 42 | if userId == "" { 43 | if cfg.UserID == "" { 44 | u, _ := user.Current() 45 | if u != nil { 46 | cfg.UserID = u.Username 47 | } 48 | } 49 | 50 | if input := prompt(fmt.Sprintf("How should packages identify you(%s): ", cfg.UserID)); len(input) > 0 { 51 | cfg.UserID = input 52 | } 53 | } 54 | 55 | if userPublicKeyPath == "" { 56 | if cfg.UserPublicKeyPath == "" { 57 | userRoot, _ := os.UserHomeDir() 58 | if userRoot != "" { 59 | cfg.UserPublicKeyPath = filepath.Join(userRoot, ".ssh", "id_rsa.pub") 60 | } 61 | } 62 | 63 | if input := prompt(fmt.Sprintf("What ssh public key should packages use (%s): ", cfg.UserPublicKeyPath)); len(input) > 0 { 64 | cfg.UserPublicKeyPath = input 65 | } 66 | } 67 | 68 | logrus.Info("installing corral, this can take a minute") 69 | 70 | if err := config.Install(); err != nil { 71 | logrus.Fatal(err) 72 | } 73 | 74 | if err := cfg.Save(); err != nil { 75 | logrus.Fatal("error saving configuration: ", err) 76 | } 77 | 78 | logrus.Info("corral installed successfully!") 79 | } 80 | 81 | func prompt(message string) string { 82 | var buf string 83 | 84 | print(message) 85 | _, _ = fmt.Scanln(&buf) 86 | 87 | return buf 88 | } 89 | -------------------------------------------------------------------------------- /cmd/config/vars/delete.go: -------------------------------------------------------------------------------- 1 | package vars 2 | 3 | import ( 4 | "github.com/rancherlabs/corral/pkg/config" 5 | "github.com/sirupsen/logrus" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | func NewCommandDelete() *cobra.Command { 10 | cmd := &cobra.Command{ 11 | Use: "delete [NAME | [NAME...]]", 12 | Short: "Remove existing global variable.", 13 | Long: "Remove existing global variable.", 14 | Args: cobra.MinimumNArgs(1), 15 | Run: deleteVar, 16 | } 17 | 18 | return cmd 19 | } 20 | 21 | func deleteVar(_ *cobra.Command, args []string) { 22 | cfg := config.MustLoad() 23 | 24 | for _, arg := range args { 25 | delete(cfg.Vars, arg) 26 | } 27 | 28 | err := cfg.Save() 29 | if err != nil { 30 | logrus.Fatalf("%e", err) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /cmd/config/vars/set.go: -------------------------------------------------------------------------------- 1 | package vars 2 | 3 | import ( 4 | "github.com/rancherlabs/corral/pkg/config" 5 | "github.com/rancherlabs/corral/pkg/vars" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | func NewCommandSet() *cobra.Command { 10 | cmd := &cobra.Command{ 11 | Use: "set NAME VALUE", 12 | Short: "Create or update global variable.", 13 | Long: "Create or update global variable.", 14 | Args: cobra.ExactArgs(2), 15 | RunE: func(cmd *cobra.Command, args []string) error { 16 | cfg := config.MustLoad() 17 | err := CreateVar(&cfg, args[0], args[1]) 18 | if err != nil { 19 | return err 20 | } 21 | return cfg.Save() 22 | }, 23 | } 24 | 25 | return cmd 26 | } 27 | 28 | func CreateVar(cfg *config.Config, key, value string) error { 29 | v, err := vars.FromJson(value) 30 | if err != nil { 31 | return err 32 | } 33 | 34 | cfg.Vars[key] = v 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /cmd/config/vars/set_test.go: -------------------------------------------------------------------------------- 1 | package vars_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | "github.com/rancherlabs/corral/cmd/config/vars" 7 | "github.com/rancherlabs/corral/pkg/config" 8 | ) 9 | 10 | var _ = Describe("Set", func() { 11 | When("a string is passed", func() { 12 | It("returns that string", func() { 13 | cfg := &config.Config{Vars: map[string]any{}} 14 | err := vars.CreateVar(cfg, "test", "test") 15 | Expect(err).To(BeNil()) 16 | Expect(cfg.Vars["test"]).To(Equal("test")) 17 | }) 18 | }) 19 | When("a number is passed", func() { 20 | It("returns a number", func() { 21 | cfg := &config.Config{Vars: map[string]any{}} 22 | err := vars.CreateVar(cfg, "test", "1") 23 | Expect(err).To(BeNil()) 24 | Expect(cfg.Vars["test"]).To(Equal(1.)) 25 | }) 26 | }) 27 | When("a number is passed in quotes", func() { 28 | It("returns a string", func() { 29 | cfg := &config.Config{Vars: map[string]any{}} 30 | err := vars.CreateVar(cfg, "test", `"1"`) 31 | Expect(err).To(BeNil()) 32 | Expect(cfg.Vars["test"]).To(Equal("1")) 33 | }) 34 | }) 35 | When("a json list is passed", func() { 36 | It("returns a slice", func() { 37 | cfg := &config.Config{Vars: map[string]any{}} 38 | err := vars.CreateVar(cfg, "test", "[1,2,3]") 39 | Expect(err).To(BeNil()) 40 | Expect(cfg.Vars["test"]).To(Equal([]any{1., 2., 3.})) 41 | }) 42 | }) 43 | When("a json object is passed", func() { 44 | It("returns a map", func() { 45 | cfg := &config.Config{Vars: map[string]any{}} 46 | err := vars.CreateVar(cfg, "test", `{"a":"1","b":2,"c":[1,2,3]}`) 47 | Expect(err).To(BeNil()) 48 | Expect(cfg.Vars["test"]).To(Equal(map[string]any{"a": "1", "b": 2., "c": []any{1., 2., 3.}})) 49 | }) 50 | }) 51 | }) 52 | -------------------------------------------------------------------------------- /cmd/config/vars/vars.go: -------------------------------------------------------------------------------- 1 | package vars 2 | 3 | import ( 4 | "fmt" 5 | 6 | pkgcmd "github.com/rancherlabs/corral/pkg/cmd" 7 | "github.com/rancherlabs/corral/pkg/config" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | var output = pkgcmd.OutputFormatTable 12 | 13 | func NewVarsCommand() *cobra.Command { 14 | cmd := &cobra.Command{ 15 | Use: "vars [VAR | [VAR...]]", 16 | Short: "List and modify global configuration.", 17 | Long: "List and modify global configuration.", 18 | RunE: func(cmd *cobra.Command, args []string) error { 19 | cfg := config.MustLoad() 20 | out, err := ListVars(cfg, output, args...) 21 | if err != nil { 22 | return err 23 | } 24 | fmt.Println(out) 25 | return nil 26 | }, 27 | } 28 | 29 | cmd.Flags().VarP(&output, "output", "o", "Output format. One of: table|json|yaml") 30 | 31 | cmd.AddCommand( 32 | NewCommandSet(), 33 | NewCommandDelete()) 34 | return cmd 35 | } 36 | 37 | func ListVars(cfg config.Config, output pkgcmd.OutputFormat, args ...string) (string, error) { 38 | if len(args) == 1 { 39 | return fmt.Sprintf("%v", cfg.Vars[args[0]]), nil 40 | } 41 | 42 | vars := map[string]any{} 43 | if len(args) > 1 { 44 | for _, k := range args { 45 | vars[k] = cfg.Vars[k] 46 | } 47 | } else { 48 | vars = cfg.Vars 49 | } 50 | 51 | return pkgcmd.Output(vars, output, pkgcmd.OutputOptions{ 52 | Key: "NAME", 53 | Value: "VALUE", 54 | }) 55 | } 56 | -------------------------------------------------------------------------------- /cmd/config/vars/vars_suite_test.go: -------------------------------------------------------------------------------- 1 | package vars_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestVars(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Vars Suite") 13 | } 14 | -------------------------------------------------------------------------------- /cmd/config/vars/vars_test.go: -------------------------------------------------------------------------------- 1 | package vars_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | "github.com/rancherlabs/corral/cmd/config/vars" 7 | pkgcmd "github.com/rancherlabs/corral/pkg/cmd" 8 | "github.com/rancherlabs/corral/pkg/config" 9 | ) 10 | 11 | var _ = Describe("Vars", func() { 12 | When("only one variable is passed", func() { 13 | When("the output format is table", func() { 14 | It("returns only one variable", func() { 15 | v, err := vars.ListVars(config.Config{Vars: map[string]any{"test": "test1"}}, pkgcmd.OutputFormatTable, "test") 16 | Expect(err).To(BeNil()) 17 | Expect(v).To(Equal("test1")) 18 | }) 19 | }) 20 | When("the output format is json", func() { 21 | It("returns only one variable", func() { 22 | v, err := vars.ListVars(config.Config{Vars: map[string]any{"test": "test1"}}, pkgcmd.OutputFormatJSON, "test") 23 | Expect(err).To(BeNil()) 24 | Expect(v).To(Equal("test1")) 25 | }) 26 | }) 27 | When("the output format is yaml", func() { 28 | It("returns only one variable", func() { 29 | v, err := vars.ListVars(config.Config{Vars: map[string]any{"test": "test1"}}, pkgcmd.OutputFormatYAML, "test") 30 | Expect(err).To(BeNil()) 31 | Expect(v).To(Equal("test1")) 32 | }) 33 | }) 34 | }) 35 | When("an unsupported output format is passed", func() { 36 | It("returns an error", func() { 37 | _, err := vars.ListVars(config.Config{Vars: map[string]any{}}, "a") 38 | Expect(err).Should(MatchError(pkgcmd.ErrUnknownOutputFormat)) 39 | }) 40 | }) 41 | When("no variables are passed", func() { 42 | When("the output format is table", func() { 43 | It("has the expected output", func() { 44 | v, err := vars.ListVars(config.Config{Vars: map[string]any{"test": "test1"}}, pkgcmd.OutputFormatTable) 45 | Expect(err).To(BeNil()) 46 | Expect(v).To(Equal("+------+-------+\n| NAME | VALUE |\n+------+-------+\n| test | test1 |\n+------+-------+")) 47 | }) 48 | }) 49 | When("the output format is json", func() { 50 | It("has the expected output", func() { 51 | v, err := vars.ListVars(config.Config{Vars: map[string]any{"test": "test1"}}, pkgcmd.OutputFormatJSON) 52 | Expect(err).To(BeNil()) 53 | Expect(v).To(Equal(`{"test":"test1"}`)) 54 | }) 55 | }) 56 | When("the output format is yaml", func() { 57 | It("has the expected output", func() { 58 | v, err := vars.ListVars(config.Config{Vars: map[string]any{"test": "test1"}}, pkgcmd.OutputFormatYAML) 59 | Expect(err).To(BeNil()) 60 | Expect(v).To(Equal("test: test1")) 61 | }) 62 | }) 63 | }) 64 | }) 65 | -------------------------------------------------------------------------------- /cmd/create.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "crypto/rsa" 7 | "crypto/x509" 8 | "encoding/pem" 9 | "os" 10 | "runtime" 11 | "strings" 12 | "sync" 13 | 14 | "github.com/pkg/errors" 15 | "github.com/rancherlabs/corral/pkg/config" 16 | "github.com/rancherlabs/corral/pkg/corral" 17 | _package "github.com/rancherlabs/corral/pkg/package" 18 | "github.com/rancherlabs/corral/pkg/shell" 19 | "github.com/rancherlabs/corral/pkg/vars" 20 | "github.com/sirupsen/logrus" 21 | "github.com/spf13/cobra" 22 | "github.com/spf13/viper" 23 | "golang.org/x/crypto/ed25519" 24 | "golang.org/x/crypto/ssh" 25 | "golang.org/x/sync/errgroup" 26 | ) 27 | 28 | var ( 29 | cfgViper = viper.New() 30 | ) 31 | 32 | const createDescription = ` 33 | Create a new corral from the given package. Packages can either be a valid OCI reference or a path to a local directory. 34 | 35 | Examples: 36 | corral create k3s ghcr.io/rancher/k3s 37 | corral create k3s-ha -v controlplane_count=3 ghcr.io/rancher/k3s 38 | corral create k3s-custom /home/rancher/issue-1234 39 | ` 40 | const ed25519KeyType = "ed25519" 41 | 42 | func NewCommandCreate() *cobra.Command { 43 | cmd := &cobra.Command{ 44 | Use: "create NAME PACKAGE", 45 | Short: "Create a new corral", 46 | Long: createDescription, 47 | Args: cobra.RangeArgs(1, 2), 48 | RunE: create, 49 | PreRun: func(cmd *cobra.Command, _ []string) { 50 | cfgFile := cmd.Flags().Lookup("config").Value.String() 51 | if cfgFile != "" { 52 | cfgViper.AddConfigPath(cfgFile) 53 | err := cfgViper.ReadInConfig() 54 | if err != nil { 55 | logrus.Fatalf("Error reading config file: %v", err) 56 | } 57 | } 58 | }, 59 | } 60 | 61 | cmd.Flags().String("config", "", "loadManifest flags for this command from a file.") 62 | 63 | cmd.Flags().StringArrayP("variable", "v", []string{}, "Set a variable to configure the package.") 64 | _ = cfgViper.BindPFlag("variable", cmd.Flags().Lookup("variable")) 65 | 66 | cmd.Flags().StringP("package", "p", "", "Set a variable to configure the package.") 67 | _ = cfgViper.BindPFlag("package", cmd.Flags().Lookup("package")) 68 | 69 | cmd.Flags().Bool("recreate", false, "Destroy corral with the same name if it exists before creating.") 70 | _ = cfgViper.BindPFlag("recreate", cmd.Flags().Lookup("recreate")) 71 | 72 | cmd.Flags().Bool("skip-cleanup", false, "Do not run terraform destroy when an error is encountered. This can result in un-tracked infrastructure resources!") 73 | _ = cfgViper.BindPFlag("skip-cleanup", cmd.Flags().Lookup("skip-cleanup")) 74 | 75 | return cmd 76 | } 77 | 78 | func create(cmd *cobra.Command, args []string) error { 79 | cfg := config.MustLoad() 80 | 81 | var corr corral.Corral 82 | corr.RootPath = config.CorralPath(args[0]) 83 | corr.Name = args[0] 84 | corr.Source = cfgViper.GetString("package") 85 | corr.NodePools = map[string][]corral.Node{} 86 | corr.Vars = map[string]any{} 87 | 88 | if len(args) > 1 { 89 | corr.Source = args[1] 90 | } 91 | // get the source from flags or args 92 | if corr.Source == "" { 93 | logrus.Fatal("You must specify a package with the `-p` flag or as an argument.") 94 | } 95 | 96 | if cfgViper.GetBool("recreate") { 97 | logrus.Infof("Deleting existing corral [%s]", args[0]) 98 | deleteCorrals(cmd, args[0:1]) 99 | } 100 | 101 | // ensure this corral is unique 102 | if corr.Exists() { 103 | logrus.Fatalf("corral [%s] already exists", corr.Name) 104 | } 105 | 106 | // load cli variables 107 | for _, raw := range cfgViper.GetStringSlice("variable") { 108 | k, v, err := vars.ToVar(raw) 109 | if err != nil { 110 | return err 111 | } 112 | if k == "" { 113 | logrus.Fatal("variables should be in the format =") 114 | } 115 | corr.Vars[k] = v 116 | } 117 | for k, v := range cfg.Vars { // copy the global vars for future reference 118 | corr.Vars[k] = v 119 | } 120 | 121 | // load the package 122 | logrus.Info("loading package") 123 | pkg, err := _package.LoadPackage(corr.Source) 124 | if err != nil { 125 | logrus.Fatalf("failed to load package: %s", err) 126 | } 127 | 128 | // update the corral ref to the absolute path 129 | corr.Source = pkg.RootPath 130 | 131 | // validate the variables 132 | err = pkg.ValidateVarSet(corr.Vars, true) 133 | if err != nil { 134 | logrus.Fatal("invalid variables: ", err) 135 | } 136 | 137 | err = pkg.ApplyDefaultVars(corr.Vars) 138 | if err != nil { 139 | logrus.Fatal("invalid defaults: ", err) 140 | } 141 | 142 | if corr.Vars["corral_private_key"] == nil && corr.Vars["corral_public_key"] == nil { 143 | logrus.Info("generating ssh keys") 144 | if corr.Vars["corral_ssh_key_type"] == ed25519KeyType { 145 | _, privkey, err := ed25519.GenerateKey(nil) 146 | if err != nil { 147 | logrus.Fatal("unable to generate private ed25519 key: ", err) 148 | } 149 | pubkey, err := ssh.NewPublicKey(privkey.Public()) 150 | if err != nil { 151 | logrus.Fatal("failed to generate public ed25519 key: ", err) 152 | } 153 | corr.PrivateKey = string(encodePrivateKeyToPEM(privkey, "OPENSSH")) 154 | corr.PublicKey = string(ssh.MarshalAuthorizedKey(pubkey)) 155 | } else { 156 | corr.Vars["corral_ssh_key_type"] = "rsa" 157 | privkey, err := generateRSAPrivateKey(2048) 158 | if err != nil { 159 | logrus.Fatal("unable to generate private rsa key: ", err) 160 | } 161 | pubkey, err := generateRSAPublicKey(&privkey.PublicKey) 162 | if err != nil { 163 | logrus.Fatal("failed to generate public rsa key: ", err) 164 | } 165 | corr.PrivateKey = string(encodePrivateKeyToPEM(privkey, "RSA")) 166 | corr.PublicKey = string(pubkey) 167 | } 168 | corr.Vars["corral_public_key"] = corr.PublicKey 169 | corr.Vars["corral_private_key"] = corr.PrivateKey 170 | } else { 171 | logrus.Info("reusing generated ssh keys") 172 | corr.PublicKey = corr.Vars["corral_public_key"].(string) 173 | corr.PrivateKey = corr.Vars["corral_private_key"].(string) 174 | } 175 | // add common variables 176 | userPublicKey, err := os.ReadFile(cfg.UserPublicKeyPath) 177 | if err != nil { 178 | logrus.Error("failed to read user public key: ", err) 179 | } 180 | corr.Vars["corral_name"] = corr.Name 181 | corr.Vars["corral_user_id"] = cfg.UserID 182 | corr.Vars["corral_user_public_key"] = string(userPublicKey) 183 | corr.Vars["corral_node_pools"] = "" 184 | 185 | // write the corral to disk 186 | corr.SetStatus(corral.StatusProvisioning) 187 | 188 | var lastCommand int 189 | knownNodes := map[*shell.Shell]struct{}{} 190 | shellRegistry := shell.NewRegistry() 191 | for i, cmd := range pkg.Manifest.Commands { 192 | lastCommand = i 193 | 194 | if cmd.Module != "" { 195 | logrus.Infof("[%d/%d] applying %s module", i+1, len(pkg.Manifest.Commands), cmd.Module) 196 | err = corr.ApplyModule(pkg.TerraformModulePath(cmd.Module), cmd.Module) 197 | if err != nil { 198 | corr.SetStatus(corral.StatusError) 199 | break 200 | } 201 | } 202 | 203 | if cmd.Parallel == nil { 204 | cmd.Parallel = &[]bool{true}[0] 205 | } 206 | 207 | if cmd.Command != "" { 208 | logrus.Infof("[%d/%d] running command %s", i+1, len(pkg.Manifest.Commands), cmd.Command) 209 | 210 | // find all distinct nodes in the given node pools 211 | var shells []*shell.Shell 212 | seen := map[*shell.Shell]struct{}{} 213 | for _, name := range cmd.NodePoolNames { 214 | if np := corr.NodePools[name]; np != nil { 215 | for _, n := range np { 216 | // get or create a shell pointer for the node 217 | sh, err := shellRegistry.GetShell(n, corr.PrivateKey, corr.Vars) 218 | if err != nil { 219 | corr.SetStatus(corral.StatusError) 220 | logrus.Errorf("failed to connect to node [%s]: %s", n.Name, err) 221 | break 222 | } 223 | 224 | // add distinct shells to the shells list 225 | if _, ok := seen[sh]; !ok { 226 | seen[sh] = struct{}{} 227 | shells = append(shells, sh) 228 | } 229 | } 230 | } 231 | } 232 | 233 | err = executeShellCommand(cmd.Command, shells, corr.Vars, *cmd.Parallel) 234 | } 235 | 236 | if err != nil { 237 | corr.SetStatus(corral.StatusError) 238 | logrus.Error(err) 239 | break 240 | } 241 | 242 | // collect new nodes to copy files 243 | var newNodeShells []*shell.Shell 244 | for npName, np := range corr.NodePools { 245 | for _, n := range np { 246 | n.OverlayRoot = pkg.Overlay[npName] 247 | sh, err := shellRegistry.GetShell(n, corr.PrivateKey, corr.Vars) 248 | if err != nil { 249 | corr.SetStatus(corral.StatusError) 250 | logrus.Errorf("failed to connect to node [%s]: %s", n.Name, err) 251 | break 252 | } 253 | 254 | if _, ok := knownNodes[sh]; !ok { 255 | newNodeShells = append(newNodeShells, sh) 256 | knownNodes[sh] = struct{}{} 257 | } 258 | } 259 | } 260 | 261 | // copy package files to new nodes 262 | err = copyPackageFiles(newNodeShells, pkg) 263 | if err != nil { 264 | corr.SetStatus(corral.StatusError) 265 | logrus.Error("failed to copy package files: ", err) 266 | break 267 | } 268 | 269 | _ = corr.Save() 270 | } 271 | 272 | // close all shells 273 | shellRegistry.Close() 274 | 275 | // if the corral is in an error state delete it 276 | if corr.Status == corral.StatusError { 277 | if cfgViper.GetBool("skip-cleanup") { 278 | logrus.Warnf("skipping roll back") 279 | _ = corr.Save() 280 | } else { 281 | logrus.Info("attempting to roll back corral") 282 | for i := lastCommand; i >= 0; i-- { 283 | if pkg.Commands[i].Module != "" { 284 | if pkg.Commands[i].SkipCleanup { 285 | continue 286 | } 287 | 288 | logrus.Infof("rolling back %s module", pkg.Commands[i].Module) 289 | if err = corr.DestroyModule(pkg.Commands[i].Module); err != nil { 290 | logrus.Fatalf("failed to cleanup module [%s]: %v", pkg.Commands[i].Module, err) 291 | } 292 | } 293 | } 294 | _ = corr.Delete() 295 | } 296 | } else { 297 | corr.SetStatus(corral.StatusReady) 298 | } 299 | 300 | logrus.Info("done!") 301 | return nil 302 | } 303 | 304 | // copyPackageFiles copies the appropriate overlay files from the given package to the shells. Concurrency is limited 305 | // to the number of cpus on the user's machine. 306 | func copyPackageFiles(shells []*shell.Shell, pkg _package.Package) error { 307 | var wg errgroup.Group 308 | sem := make(chan bool, runtime.NumCPU()) 309 | 310 | for _, sh := range shells { 311 | sh := sh 312 | wg.Go(func() error { 313 | sem <- true 314 | 315 | err := sh.UploadPackageFiles(pkg) 316 | 317 | <-sem 318 | return err 319 | }) 320 | } 321 | 322 | return wg.Wait() 323 | } 324 | 325 | func executeShellCommand(command string, shells []*shell.Shell, vs vars.VarSet, parallel bool) error { 326 | var err error 327 | if parallel { 328 | err = executeShellCommandAsync(command, shells, vs) 329 | } else { 330 | err = executeShellCommandSync(command, shells, vs) 331 | } 332 | if err != nil { 333 | return errors.Wrapf(err, "running %s", command) 334 | } 335 | return nil 336 | } 337 | 338 | // executeShellCommandAsync runs the given command on the given shells. Any vars set are saved to the VarSet. 339 | // Concurrency is limited to the number of cpus on the user's machine. 340 | func executeShellCommandAsync(command string, shells []*shell.Shell, vs vars.VarSet) error { 341 | var mu sync.Mutex 342 | var wg errgroup.Group 343 | sem := make(chan bool, runtime.NumCPU()) 344 | 345 | for _, sh := range shells { 346 | sh := sh 347 | wg.Go(func() error { 348 | sem <- true 349 | 350 | err := sh.Run(command) 351 | if err != nil { 352 | <-sem 353 | return err 354 | } 355 | 356 | mu.Lock() 357 | for k, v := range sh.Vars { 358 | vs[k] = v 359 | } 360 | mu.Unlock() 361 | 362 | <-sem 363 | return nil 364 | }) 365 | } 366 | 367 | return wg.Wait() 368 | } 369 | 370 | func executeShellCommandSync(command string, shells []*shell.Shell, vs vars.VarSet) error { 371 | for _, sh := range shells { 372 | sh := sh 373 | err := sh.Run(command) 374 | if err != nil { 375 | return err 376 | } 377 | 378 | for k, v := range sh.Vars { 379 | vs[k] = v 380 | } 381 | } 382 | 383 | return nil 384 | } 385 | 386 | func generateRSAPrivateKey(bits int) (*rsa.PrivateKey, error) { 387 | privateKey, err := rsa.GenerateKey(rand.Reader, bits) 388 | if err != nil { 389 | return nil, err 390 | } 391 | 392 | err = privateKey.Validate() 393 | if err != nil { 394 | return nil, err 395 | } 396 | 397 | return privateKey, nil 398 | } 399 | 400 | func generateRSAPublicKey(key *rsa.PublicKey) ([]byte, error) { 401 | publicRsaKey, err := ssh.NewPublicKey(key) 402 | if err != nil { 403 | return nil, err 404 | } 405 | 406 | pubKeyBytes := bytes.Replace(ssh.MarshalAuthorizedKey(publicRsaKey), []byte("\n"), []byte(""), 2) 407 | 408 | return pubKeyBytes, nil 409 | } 410 | 411 | func encodePrivateKeyToPEM(key any, blockType string) []byte { 412 | blockTypeDefault := "PRIVATE KEY" 413 | 414 | if len(blockType) > 0 { 415 | blockTypeDefault = " " + blockTypeDefault 416 | } 417 | blockType = blockType + blockTypeDefault 418 | 419 | var privDER []byte 420 | var err error 421 | if strings.Contains(blockType, "OPENSSH") { 422 | // Necessary to cast type in order to correctly marshal the key for OpenSSH 423 | privDER, err = x509.MarshalPKCS8PrivateKey(key.(ed25519.PrivateKey)) 424 | } else if strings.Contains(blockType, "RSA") { 425 | privDER = x509.MarshalPKCS1PrivateKey(key.(*rsa.PrivateKey)) 426 | } else { 427 | privDER, err = x509.MarshalPKCS8PrivateKey(key) 428 | } 429 | 430 | if err != nil { 431 | logrus.Fatal("failed to marshal PKCS8 private key: ", err) 432 | } 433 | 434 | privBlock := pem.Block{ 435 | Type: blockType, 436 | Headers: nil, 437 | Bytes: privDER, 438 | } 439 | 440 | return pem.EncodeToMemory(&privBlock) 441 | } 442 | -------------------------------------------------------------------------------- /cmd/delete.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | 7 | "github.com/rancherlabs/corral/pkg/config" 8 | "github.com/rancherlabs/corral/pkg/corral" 9 | _package "github.com/rancherlabs/corral/pkg/package" 10 | "github.com/sirupsen/logrus" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | const deleteDescription = ` 15 | Delete the given corral(s) and the associated infrastructure. If multiple corrals are given they will be deleted in 16 | the order they appear one at a time. 17 | ` 18 | 19 | func NewCommandDelete() *cobra.Command { 20 | cmd := &cobra.Command{ 21 | Use: "delete [NAME [NAME ...]", 22 | Short: "Delete the given corral(s) and the associated infrastructure.", 23 | Long: deleteDescription, 24 | Args: cobra.MinimumNArgs(1), 25 | Run: deleteCorrals, 26 | } 27 | 28 | cmd.Flags().Bool("skip-cleanup", false, "Do not run terraform destroy just delete the package. This can result in un-tracked infrastructure resources!") 29 | 30 | return cmd 31 | } 32 | 33 | func deleteCorrals(cmd *cobra.Command, args []string) { 34 | skipCleanup, _ := cmd.Flags().GetBool("skip-cleanup") 35 | for _, name := range args { 36 | err := deleteCorral(name, skipCleanup) 37 | if err != nil { 38 | logrus.Errorf("failed to delete corral [%s]: %s", name, err) 39 | continue 40 | } 41 | logrus.Infof("deleted corral [%s]", name) 42 | } 43 | } 44 | 45 | func deleteCorral(name string, skipCleanup bool) error { 46 | c, err := corral.Load(config.CorralPath(name)) 47 | if err != nil { 48 | if errors.Is(err, os.ErrNotExist) { 49 | logrus.Warnf("skipping corral [%s], does not exist", name) 50 | return nil 51 | } else { 52 | return err 53 | } 54 | } 55 | 56 | c.SetStatus(corral.StatusDeleting) 57 | 58 | if !skipCleanup { 59 | logrus.Infof("cleaning up corral: %s", name) 60 | pkg, err := _package.LoadPackage(c.Source) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | for i := len(pkg.Commands) - 1; i >= 0; i-- { 66 | if pkg.Commands[i].Module != "" { 67 | if pkg.Commands[i].SkipCleanup { 68 | continue 69 | } 70 | 71 | logrus.Debugf("destroying module: %s", pkg.Commands[i].Module) 72 | if err = c.DestroyModule(pkg.Commands[i].Module); err != nil { 73 | logrus.Errorf("failed to cleanup module [%s]: %v", pkg.Commands[i].Module, err) 74 | continue 75 | } 76 | } 77 | } 78 | } else { 79 | logrus.Warnf("skipping cleanup for corral [%s]", name) 80 | } 81 | 82 | return c.Delete() 83 | } 84 | -------------------------------------------------------------------------------- /cmd/list.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | pkgcmd "github.com/rancherlabs/corral/pkg/cmd" 8 | "github.com/rancherlabs/corral/pkg/config" 9 | "github.com/rancherlabs/corral/pkg/corral" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func NewCommandList() *cobra.Command { 14 | cmd := &cobra.Command{ 15 | Use: "list", 16 | Short: "List all corrals on this system.", 17 | Long: "List all corrals on this system.", 18 | RunE: list, 19 | } 20 | 21 | cmd.Flags().VarP(&output, "output", "o", "Output format. One of: table|json|yaml") 22 | 23 | return cmd 24 | } 25 | 26 | func list(_ *cobra.Command, _ []string) error { 27 | corralNames, _ := os.ReadDir(config.CorralRoot("corrals")) 28 | 29 | corrals := map[string]string{} 30 | for _, entry := range corralNames { 31 | c, err := corral.Load(config.CorralRoot("corrals", entry.Name())) 32 | if err != nil { 33 | continue 34 | } 35 | 36 | corrals[c.Name] = c.Status.String() 37 | } 38 | 39 | out, err := pkgcmd.Output(corrals, output, pkgcmd.OutputOptions{ 40 | Key: "NAME", 41 | Value: "STATUS", 42 | }) 43 | if err != nil { 44 | return err 45 | } 46 | fmt.Println(out) 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /cmd/package/download.go: -------------------------------------------------------------------------------- 1 | package cmd_package 2 | 3 | import ( 4 | "io" 5 | "io/fs" 6 | "os" 7 | "path/filepath" 8 | 9 | _package "github.com/rancherlabs/corral/pkg/package" 10 | "github.com/sirupsen/logrus" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | const downloadDescription = ` 15 | Download a package from an OCI registry to the local filesystem. 16 | 17 | Examples: 18 | corral package download ghcr.io/rancher/my_pkg:latest 19 | corral package download ghcr.io/rancher/my_pkg:latest dest 20 | ` 21 | 22 | func NewCommandDownload() *cobra.Command { 23 | cmd := &cobra.Command{ 24 | Use: "download PACKAGE [DEST]", 25 | Short: "Download a package from an OCI registry.", 26 | Long: downloadDescription, 27 | Run: download, 28 | Args: cobra.RangeArgs(1, 2), 29 | } 30 | 31 | return cmd 32 | } 33 | 34 | func download(_ *cobra.Command, args []string) { 35 | pkg, err := _package.LoadPackage(args[0]) 36 | if err != nil { 37 | logrus.Fatalf("failed to load package: %s", err) 38 | } 39 | 40 | dest := pkg.Name 41 | if len(args) > 1 { 42 | dest = args[1] 43 | } 44 | 45 | err = filepath.WalkDir(pkg.RootPath, func(path string, d fs.DirEntry, err error) error { 46 | if err != nil { 47 | return err 48 | } 49 | 50 | destPath := dest + path[len(pkg.RootPath):] 51 | 52 | if d.IsDir() { 53 | if err := os.Mkdir(destPath, 0700); err != nil { 54 | return err 55 | } 56 | } else { 57 | f, err := os.Create(destPath) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | in, err := os.Open(path) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | _, err = io.Copy(f, in) 68 | f.Close() 69 | in.Close() 70 | 71 | if err != nil { 72 | return err 73 | } 74 | } 75 | 76 | return nil 77 | }) 78 | 79 | if err != nil { 80 | logrus.Fatalf("failed to copy package files to destination: %s", err) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /cmd/package/info.go: -------------------------------------------------------------------------------- 1 | package cmd_package 2 | 3 | import ( 4 | _package "github.com/rancherlabs/corral/pkg/package" 5 | "github.com/sirupsen/logrus" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | func NewCommandInfo() *cobra.Command { 10 | cmd := &cobra.Command{ 11 | Use: "info PACKAGE", 12 | Short: "Display details about the given package.", 13 | Args: cobra.ExactArgs(1), 14 | Run: info, 15 | } 16 | 17 | return cmd 18 | } 19 | 20 | func info(_ *cobra.Command, args []string) { 21 | pkg, err := _package.LoadPackage(args[0]) 22 | if err != nil { 23 | logrus.Fatal(err) 24 | } 25 | 26 | println(pkg.Name) 27 | println() 28 | println(pkg.Description) 29 | println() 30 | println("VARIABLE\tDESCRIPTION") 31 | for k, v := range pkg.VariableSchemas { 32 | println(k, "\t", v.Description) 33 | } 34 | println() 35 | } 36 | -------------------------------------------------------------------------------- /cmd/package/login.go: -------------------------------------------------------------------------------- 1 | package cmd_package 2 | 3 | import ( 4 | "fmt" 5 | 6 | _package "github.com/rancherlabs/corral/pkg/package" 7 | "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | func NewCommandLogin() *cobra.Command { 12 | cmd := &cobra.Command{ 13 | Use: "login REGISTRY", 14 | Short: "Login to an OCI registry.", 15 | Args: cobra.ExactArgs(1), 16 | Run: login, 17 | } 18 | 19 | cmd.Flags().String("username", "", "The username for the registry.") 20 | cmd.Flags().String("password", "", "The password for the user.") 21 | 22 | return cmd 23 | } 24 | 25 | func login(cmd *cobra.Command, args []string) { 26 | username, _ := cmd.Flags().GetString("username") 27 | password, _ := cmd.Flags().GetString("password") 28 | 29 | if username == "" { 30 | username = prompt("username: ") 31 | } 32 | 33 | if password == "" { 34 | password = prompt("password: ") 35 | } 36 | 37 | err := _package.AddRegistryCredentials(args[0], username, password) 38 | if err != nil { 39 | logrus.Fatalf("error authenticating with registry: %s", err) 40 | } 41 | 42 | logrus.Info("success!") 43 | } 44 | 45 | func prompt(message string) string { 46 | var buf string 47 | 48 | print(message) 49 | _, _ = fmt.Scanln(&buf) 50 | 51 | return buf 52 | } 53 | -------------------------------------------------------------------------------- /cmd/package/package.go: -------------------------------------------------------------------------------- 1 | package cmd_package 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | func NewCommandPackage() *cobra.Command { 9 | cmd := &cobra.Command{ 10 | Use: "package", 11 | Short: "Commands related to managing packages.", 12 | Run: func(cmd *cobra.Command, args []string) { 13 | if err := cmd.Usage(); err != nil { 14 | logrus.Fatalln(err) 15 | } 16 | }, 17 | } 18 | 19 | cmd.AddCommand( 20 | NewCommandPublish(), 21 | NewCommandLogin(), 22 | NewCommandInfo(), 23 | NewCommandValidate(), 24 | NewCommandDownload(), 25 | NewCommandTemplate()) 26 | return cmd 27 | } 28 | -------------------------------------------------------------------------------- /cmd/package/publish.go: -------------------------------------------------------------------------------- 1 | package cmd_package 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/rancherlabs/corral/pkg/config" 7 | _package "github.com/rancherlabs/corral/pkg/package" 8 | "github.com/rancherlabs/corral/pkg/version" 9 | "github.com/sirupsen/logrus" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | const publishDescription = ` 14 | Upload the package found at the given target to the given registry. 15 | 16 | Examples: 17 | corral publish /home/rancher/my_pkg ghcr.io/rancher/my_pkg:latest 18 | ` 19 | 20 | func NewCommandPublish() *cobra.Command { 21 | cmd := &cobra.Command{ 22 | Use: "publish TARGET REFERENCE", 23 | Short: "Upload a package to an OCI registry.", 24 | Long: publishDescription, 25 | Run: publish, 26 | Args: cobra.ExactArgs(2), 27 | } 28 | 29 | return cmd 30 | } 31 | 32 | func publish(_ *cobra.Command, args []string) { 33 | cfg := config.MustLoad() 34 | 35 | pkg, err := _package.LoadPackage(args[0]) 36 | if err != nil { 37 | logrus.Fatal("failed to load package: ", err) 38 | } 39 | 40 | setAnnotationIfEmpty(pkg.Annotations, _package.TerraformVersionAnnotation, version.TerraformVersion) 41 | setAnnotationIfEmpty(pkg.Annotations, _package.PublisherAnnotation, cfg.UserID) 42 | setAnnotationIfEmpty(pkg.Annotations, _package.CorralVersionAnnotation, version.Version) 43 | setAnnotationIfEmpty(pkg.Annotations, _package.PublishTimestampAnnotation, time.Now().UTC().Format(time.RFC3339)) 44 | 45 | err = _package.UploadPackage(pkg, args[1]) 46 | if err != nil { 47 | logrus.Fatal("failed to push package: ", err) 48 | } 49 | 50 | logrus.Info("success") 51 | } 52 | 53 | func setAnnotationIfEmpty(annotations map[string]string, key, value string) { 54 | if annotations != nil && annotations[key] == "" { 55 | annotations[key] = value 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /cmd/package/template.go: -------------------------------------------------------------------------------- 1 | package cmd_package 2 | 3 | import ( 4 | _package "github.com/rancherlabs/corral/pkg/package" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | const templateDescription = ` 9 | Create a package from existing package(s). 10 | 11 | Examples: 12 | corral package template a b c OUT 13 | corral package template --description "my description" a b c OUT 14 | ` 15 | 16 | func NewCommandTemplate() *cobra.Command { 17 | var description string 18 | 19 | cmd := &cobra.Command{ 20 | Use: "template PACKAGE[S] NAME", 21 | Short: "Create a package from a template", 22 | Long: templateDescription, 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | return _package.Template(args[len(args)-1], description, args[:len(args)-1]...) 25 | }, 26 | Args: cobra.MinimumNArgs(2), 27 | } 28 | 29 | cmd.Flags().StringVarP(&description, "description", "d", "", "description of the rendered package") 30 | 31 | return cmd 32 | } 33 | -------------------------------------------------------------------------------- /cmd/package/validate.go: -------------------------------------------------------------------------------- 1 | package cmd_package 2 | 3 | import ( 4 | _package "github.com/rancherlabs/corral/pkg/package" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | func NewCommandValidate() *cobra.Command { 9 | cmd := &cobra.Command{ 10 | Use: "validate PACKAGE", 11 | Short: "Validate the given package's manifest and structure.", 12 | Args: cobra.ExactArgs(1), 13 | RunE: func(_ *cobra.Command, args []string) error { 14 | return _package.Validate(args[0]) 15 | }, 16 | } 17 | 18 | return cmd 19 | } 20 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/rancherlabs/corral/cmd/config" 5 | 6 | cmdpackage "github.com/rancherlabs/corral/cmd/package" 7 | pkgcmd "github.com/rancherlabs/corral/pkg/cmd" 8 | "github.com/rancherlabs/corral/pkg/version" 9 | "github.com/sirupsen/logrus" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | var output = pkgcmd.OutputFormatTable 14 | 15 | func Execute() { 16 | var debug bool 17 | var trace bool 18 | 19 | rootCmd := &cobra.Command{ 20 | Use: "corral", 21 | Short: "Corral is a CLI tool for creating and packaging reproducible development environments.", 22 | Long: "Corral is a CLI tool for creating and packaging reproducible development environments.", 23 | Version: version.Version, 24 | PersistentPreRun: func(cmd *cobra.Command, args []string) { 25 | if trace { 26 | logrus.SetLevel(logrus.TraceLevel) 27 | } else if debug { 28 | logrus.SetLevel(logrus.DebugLevel) 29 | } 30 | }, 31 | Run: func(cmd *cobra.Command, args []string) { 32 | if err := cmd.Usage(); err != nil { 33 | logrus.Fatalln(err) 34 | } 35 | }, 36 | SilenceUsage: true, 37 | SilenceErrors: true, 38 | } 39 | 40 | rootCmd.AddCommand( 41 | config.NewCommandConfig(), 42 | NewCommandDelete(), 43 | NewCommandList(), 44 | NewCommandVars(), 45 | NewCommandCreate(), 46 | cmdpackage.NewCommandPackage()) 47 | 48 | rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Enable verbose logging") 49 | rootCmd.PersistentFlags().BoolVar(&trace, "trace", false, "Enable verboser logging") 50 | 51 | if err := rootCmd.Execute(); err != nil { 52 | logrus.Fatalln(err) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /cmd/vars.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | pkgcmd "github.com/rancherlabs/corral/pkg/cmd" 8 | "github.com/rancherlabs/corral/pkg/config" 9 | "github.com/rancherlabs/corral/pkg/corral" 10 | _package "github.com/rancherlabs/corral/pkg/package" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | const varsDescription = ` 15 | Show the given corral's variables. If not variable is specified all variables are returned. If one variables 16 | is specified only that variables value is returned. If multiple variables are specified they will be returned as a table. 17 | 18 | Examples: 19 | 20 | corral vars k3s 21 | corral vars k3s kube_api_host node_token 22 | corral vars k3s kubeconfig | base64 --decode > ~/.kube/config 23 | corral vars k3s -a 24 | ` 25 | 26 | func NewCommandVars() *cobra.Command { 27 | cmd := &cobra.Command{ 28 | Use: "vars NAME [VAR | [VAR...]]", 29 | Short: "Show the given corral's variables", 30 | Long: varsDescription, 31 | RunE: listVars, 32 | Args: cobra.MinimumNArgs(1), 33 | } 34 | 35 | cmd.Flags().Bool("sensitive", false, "Sensitive values will be displayed if this flag is true.") 36 | cmd.Flags().VarP(&output, "output", "o", "Output format. One of: table|json|yaml") 37 | cmd.Flags().BoolP("all", "a", false, "All values will be displayed if this flag is true.") 38 | 39 | return cmd 40 | } 41 | 42 | func listVars(cmd *cobra.Command, args []string) error { 43 | corralName := args[0] 44 | 45 | c, err := corral.Load(config.CorralPath(corralName)) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | // if only one output is requested return the raw value 51 | if len(args) == 2 { 52 | _, _ = os.Stdout.WriteString(fmt.Sprintf("%v\n", c.Vars[args[1]])) 53 | return nil 54 | } 55 | 56 | pkg, err := _package.LoadPackage(c.Source) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | vs := c.Vars 62 | 63 | if all, _ := cmd.Flags().GetBool("all"); !all { 64 | vs = pkg.FilterVars(vs) 65 | } 66 | 67 | if sensitive, _ := cmd.Flags().GetBool("sensitive"); !sensitive { 68 | vs = pkg.FilterSensitiveVars(vs) 69 | } 70 | 71 | out, err := pkgcmd.Output(vs, output, pkgcmd.OutputOptions{ 72 | Key: "NAME", 73 | Value: "VALUE", 74 | }) 75 | if err != nil { 76 | return err 77 | } 78 | fmt.Println(out) 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /docs/packages/tutorial.md: -------------------------------------------------------------------------------- 1 | # How to Create a Package 2 | 3 | For this tutorial we will create a Corral package that sets up an OCI registry with authentication on a Digitalocean 4 | droplet. 5 | 6 | # Anatomy of a Package 7 | 8 | To create a corral package we need to understand what goes into a package. Packages consist of 3 components. The manifest 9 | defines what information is displayed to end users and what the user is required to provide to use the package. The 10 | terraform modules define the infrastructure required for this package. Finally, the overlay defines scripts and assets 11 | used to configure the applications running on the package infrastructure. 12 | 13 | To start lets define the basic folder structure of our package. Going forward we will refer to the registry folder as 14 | the root of our package. 15 | 16 | ```shell 17 | mkdir -p registry/{terraform/main,overlay} 18 | ``` 19 | 20 | # Defining the Manifest 21 | 22 | The manifest tells corral how to create a corral from this package as well as how users should interact with it. To start 23 | lets give our package a name and description. Create `manifest.yaml` in our package's root directory. Our description 24 | should describe what the package does as well as give the user a sense of what the package will create and how it should 25 | be used. Packages should be specific in their usage to make them as reproducible as possible. It is better to create 26 | many similar packages than a single customizable package. 27 | 28 | ```yaml 29 | name: registry 30 | description: > 31 | An authenticated docker registry running in docker for local development. 32 | ``` 33 | 34 | Now we can define how users will interact with this package. We can do this with variables. variables define what values 35 | can be passed into a package as well as what values should be displayed to a user. While corral scripts and terraform 36 | modules can define any variable they like, only the variables defined in the manifest will be exposed to the user. Lets 37 | update our `manifest.yaml` to look like this. 38 | 39 | ```yaml 40 | name: registry 41 | description: | 42 | An authenticated docker registry running in Digitalocean. 43 | variables: 44 | digitalocean_token: 45 | sensitive: true 46 | type: string 47 | optional: false 48 | description: "A Digitalocean API token with write permission. https://docs.digitalocean.com/reference/api/create-personal-access-token/" 49 | digitalocean_domain: 50 | sensitive: true 51 | type: string 52 | optional: false 53 | description: "The domain to use for the registry host." 54 | registry_host: 55 | type: string 56 | readOnly: true 57 | description: "host the configured registry can be accessed at" 58 | username: 59 | type: string 60 | readOnly: true 61 | description: "username for registry authentication" 62 | password: 63 | type: string 64 | readOnly: true 65 | description: "password for registry authentication" 66 | ``` 67 | 68 | As we can see the package requires Digitalocean credentials and a domain. While these values may be different uesr to 69 | user the will not fundamentally change the behavior of the package. Any variables that change behavior should be 70 | a different package. We also define `registry_host`, `username`, and `password` These values will be defined by Corral 71 | when the package is used and cannot be changed by the user. 72 | 73 | # Writing a Terraform Module 74 | 75 | Now that we have defined how we want our package to work lets define the infrastructure we need to make it happen. To start 76 | lets create a file with all the variables available to us. Any Corral variable will be exposed as a terraform variable 77 | prefixed with `corral_`. 78 | 79 | Create the file `terraform/main/corral.tf`. It is best practice to define corral variables in `corral.tf` 80 | ```terraform 81 | // Corral 82 | variable "corral_name" {} // name of the corral being created 83 | variable "corral_user_id" {} // how the user is identified (usually github username) 84 | variable "corral_user_public_key" {} // the users public key 85 | variable "corral_public_key" {} // The corrals public key. This should be installed on every node. 86 | variable "corral_private_key" {} // The corrals private key. 87 | 88 | // Package 89 | variable "digitalocean_token" {} 90 | variable "digitalocean_domain" {} 91 | ``` 92 | 93 | Now that we have defined what values are available we can create our registry's infrastructure. 94 | 95 | `terraform/main/main.tf` 96 | 97 | ```terraform 98 | terraform { 99 | required_version = ">= 0.13" 100 | required_providers { 101 | digitalocean = { 102 | source = "digitalocean/digitalocean" 103 | version = "~> 2.0" 104 | } 105 | } 106 | } 107 | 108 | provider "random" {} 109 | provider "digitalocean" { 110 | token = var.digitalocean_token 111 | } 112 | 113 | // it is best practice to distinguish an environment with a random id to avoid collisions 114 | resource "random_id" "registry_id" { 115 | byte_length = 6 116 | } 117 | 118 | // we will use the corral public key to get access to nodes to provision them later 119 | resource "digitalocean_ssh_key" "corral_key" { 120 | name = "${var.corral_user_id}-${random_id.registry_id.hex}" 121 | public_key = var.corral_public_key 122 | } 123 | 124 | resource "digitalocean_droplet" "registry" { 125 | count = 1 126 | 127 | name = "${var.corral_user_id}-${random_id.registry_id.hex}-registry" 128 | image = "ubuntu-20-04-x64" 129 | region = "sfo3" 130 | size = "s-1vcpu-2gb" 131 | tags = [var.corral_user_id, random_id.registry_id.hex] // when possible resources should be marked with the associated corral 132 | ssh_keys = [digitalocean_ssh_key.corral_key.id] 133 | } 134 | 135 | resource "digitalocean_record" "registry" { 136 | domain = var.digitalocean_domain 137 | name = random_id.registry_id.hex 138 | type = "A" 139 | value = digitalocean_droplet.registry[0].ipv4_address 140 | } 141 | ``` 142 | 143 | Now that we have all this infrastructure we need to tell corral how to interact with our infrastructure. We need 144 | to define our node pools. Node pools are grouping of ssh hosts that corral can execute commands on. In addition to 145 | the node pools we can set the registry host here as we have everything we need to define it. Any terraform output will 146 | be stored as a corral variable. 147 | 148 | Let's create `terraform/main/outputs.tf` 149 | 150 | ```terraform 151 | output "corral_node_pools" { 152 | value = { 153 | registry = [ 154 | for droplet in digitalocean_droplet.registry : { 155 | name = droplet.name // unique name of node 156 | user = "root" // ssh username 157 | address = droplet.ipv4_address // address of ssh host 158 | } 159 | ] 160 | } 161 | } 162 | 163 | output "registry_host" { 164 | value = join(".", [digitalocean_record.registry.name, digitalocean_record.registry.domain]) 165 | } 166 | ``` 167 | 168 | # Overlay 169 | 170 | Now that we have some infrastructure to work with we can configure our application. By default, the overlay directory 171 | will be copied to the root directory of all nodes. All files will be copied with the ownership of the ssh user in mode 172 | `0777`. Best practice is to put any scripts used only for provisioning the nodes in `/opt/corral`. For the purposes 173 | of this tutorial we can just copy the overlay directory from `examples/registry/overlay` in this repository. This 174 | contains the registry binary and some other assets need for the application. Most of these files do not interact with 175 | corral but `overlay/opt/corral/install.sh` takes advantage of some Corral shell features. 176 | 177 | ```shell 178 | #!/bin/bash 179 | set -ex 180 | 181 | # corral_set allows us to set corral variables from scripts. 182 | function corral_set() { 183 | echo "corral_set $1=$2" 184 | } 185 | 186 | # corral_log allows us to print messages for the corral user. 187 | function corral_log() { 188 | echo "corral_log $1" 189 | } 190 | 191 | # Install the user's public key incase they need to debug an issue 192 | echo "$CORRAL_corral_user_public_key" >> /$(whoami)/.ssh/authorized_keys 193 | 194 | apt install -y apache2-utils 195 | 196 | 197 | USERNAME="corral" 198 | PASSWORD="$( echo $RANDOM | md5sum | head -c 12)" # it is best practice to generate passwords for every distinct corral 199 | 200 | # here we set the username and password for the user to find later 201 | corral_set username $USERNAME 202 | corral_set password $PASSWORD 203 | 204 | # this will be used by the docker registry for authentication 205 | htpasswd -Bbn $USERNAME "$PASSWORD" > /etc/docker/registry/htpasswd 206 | 207 | # corral variables are available as environment variables with the prefix `CORRAL_` 208 | sed -i "s/HOSTNAME/$CORRAL_registry_host/g" /etc/docker/registry/config.yml 209 | 210 | # generate self signed certificates 211 | openssl req -x509 \ 212 | -newkey rsa:4096 \ 213 | -sha256 \ 214 | -days 3650 \ 215 | -nodes \ 216 | -keyout /etc/docker/registry/ssl/registry.key \ 217 | -out /etc/docker/registry/ssl/registry.crt \ 218 | -subj "/CN=${CORRAL_registry_host}" \ 219 | -addext "subjectAltName=DNS:${CORRAL_registry_host}" 220 | 221 | corral_log "This registry uses self signed certificates please add {\"insecure_registries\":[\"${CORRAL_registry_host}\"]} to /etc/docker/daemon.json." 222 | 223 | systemctl enable registry 224 | systemctl start registry 225 | ``` 226 | 227 | # Commands 228 | 229 | The last step is to tell corral to run our terraform module in the manifest. We do this with the commands section. 230 | Commands can either be a terraform module or a shell command to run on a node pool. If there are multiple nodes in a 231 | node pool the commands will be run concurrently. To run our terraform module let's add a command section to our 232 | `manifest.yaml` 233 | 234 | ```yaml 235 | commands: 236 | - module: main 237 | ``` 238 | 239 | Now that our terraform is run we can run commands against our registry node to configure the registry. You can have as 240 | commands as you want but for this package we only need to run a single install script. 241 | 242 | ```yaml 243 | commands: 244 | - module: main 245 | - command: /opt/corral/install.sh 246 | node_pools: 247 | - registry 248 | ``` 249 | 250 | # Validating a Package 251 | 252 | At this point we have configured our manifest, infrastructure and scripts to configure our application. We should now 253 | have a valid corral package that we can create corrals from! Before we try to create a corral we can validate our 254 | package. 255 | 256 | ```shell 257 | corral package validate ./registry 258 | ``` 259 | 260 | If we have any typos in our manifest or the folder structure has any problems this command will output them. 261 | 262 | 263 | # Installing a Local Package 264 | 265 | Assuming our package validated we can now test it! If there are any issues corral will automatically rollback the 266 | infrastructure. This is convenient for keeping cloud environments clean but can make it difficult to diagnose issues. 267 | We can tell corral to pipe stdout and stdin to our terminal while we create our package to better understand any issues 268 | we may encounter with the `--debug` flag. We can also pass our Digitalocean credentials and domain with the `-v` flag. 269 | For variables like these it is better to set them as global variables, so we don't need to type them out every time we 270 | create a corral from a package that uses Digitalocean. 271 | 272 | ```shell 273 | corral config vars set digitalocean_token MY_DO_TOKEN 274 | corral config vars set digitalocean_domain my.domain.example.com 275 | ``` 276 | 277 | ```shell 278 | corral create registry ./registry --debug 279 | ``` 280 | 281 | Once our package is created we can see that our registry host and credentials are ready to use. 282 | ```shell 283 | corral vars registry 284 | ``` 285 | -------------------------------------------------------------------------------- /examples/complex-types/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: complex-types 2 | description: > 3 | A simple package for developing corral. 4 | commands: 5 | - module: module 6 | - command: /app/complex.sh 7 | node_pools: 8 | - all 9 | variables: 10 | mapvar: 11 | type: object 12 | default: 13 | foo: bar 14 | description: "Example json object variable for testing variable flows." 15 | mapvar1: 16 | type: object 17 | readOnly: true 18 | mapvar2: 19 | type: object 20 | readOnly: true 21 | listvar: 22 | type: array 23 | default: 24 | - 1 25 | - 2 26 | - 3 27 | description: "Example json array variable for testing variable flows." 28 | listvar1: 29 | type: array 30 | readOnly: true 31 | listvar2: 32 | type: array 33 | readOnly: true 34 | numbervar: 35 | type: number 36 | default: 1 37 | description: "Example json number variable for testing variable flows." 38 | stringvar: 39 | type: string 40 | readOnly: true -------------------------------------------------------------------------------- /examples/complex-types/overlay/app/complex.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | CORRAL_listvar="${CORRAL_listvar}" 5 | CORRAL_mapvar="${CORRAL_mapvar}" 6 | 7 | echo listvar ${CORRAL_listvar} 8 | echo mapvar ${CORRAL_mapvar} 9 | 10 | echo "corral_set listvar2=${CORRAL_listvar}" 11 | echo "corral_set mapvar2=${CORRAL_mapvar}" 12 | 13 | echo "corral_set numbervar=2" 14 | echo "corral_set stringvar=abc" -------------------------------------------------------------------------------- /examples/complex-types/terraform/module/corral.tf: -------------------------------------------------------------------------------- 1 | variable "corral_name" {} 2 | variable "corral_public_key" {} 3 | variable "mapvar" { 4 | type = map(string) 5 | } 6 | variable "listvar" { 7 | type = list(number) 8 | } 9 | 10 | output "corral_node_pools" { 11 | value = { 12 | all = [ 13 | for n in docker_container.node : { 14 | name = n.name 15 | user = "corral" 16 | address = "127.0.0.1:${n.ports[0].external}" 17 | } 18 | ] 19 | } 20 | } 21 | 22 | output "mapvar1" { 23 | value = var.mapvar 24 | } 25 | 26 | output "listvar1" { 27 | value = var.listvar 28 | } -------------------------------------------------------------------------------- /examples/complex-types/terraform/module/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | docker = { 4 | source = "kreuzwerker/docker" 5 | version = "~> 2.13.0" 6 | } 7 | } 8 | } 9 | 10 | provider "docker" {} 11 | 12 | resource "docker_volume" "data" { 13 | count = 1 14 | name = "${var.corral_name}-node-${count.index}" 15 | } 16 | 17 | resource "docker_container" "node" { 18 | count = 1 19 | image = "lscr.io/linuxserver/openssh-server" 20 | name = "${var.corral_name}-node-${count.index}" 21 | 22 | ports { 23 | internal = 2222 24 | } 25 | 26 | env = [ 27 | "PUBLIC_KEY=${var.corral_public_key}", 28 | "USER_NAME=corral", 29 | "USER_PASSWORD=corral", 30 | "SUDO_ACCESS=true", 31 | "PASSWORD_ACCESS=true", 32 | ] 33 | 34 | volumes { 35 | container_path = "/app" 36 | volume_name = docker_volume.data[count.index].name 37 | } 38 | } -------------------------------------------------------------------------------- /examples/multistage/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: simple 2 | description: > 3 | A simple package for developing corral. 4 | commands: 5 | - module: first 6 | - command: ls /app 7 | node_pools: 8 | - all 9 | - module: second 10 | - command: ls /app 11 | node_pools: 12 | - all 13 | variables: 14 | var1: 15 | type: string 16 | default: "foo" 17 | description: "Example variable for testing variable flows." 18 | var1_out: 19 | type: string 20 | readOnly: true 21 | description: "Set to the value of var1 by a corral set script." -------------------------------------------------------------------------------- /examples/multistage/overlay/app/setvar1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | env 4 | 5 | echo "corral_set var1_out=${CORRAL_var1}" 6 | -------------------------------------------------------------------------------- /examples/multistage/terraform/first/corral.tf: -------------------------------------------------------------------------------- 1 | variable "corral_name" {} 2 | variable "corral_public_key" {} 3 | 4 | output "corral_node_pools" { 5 | value = { 6 | all = [ 7 | for n in docker_container.node : { 8 | name = n.name 9 | user = "corral" 10 | address = "127.0.0.1:${n.ports[0].external}" 11 | } 12 | ] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/multistage/terraform/first/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | docker = { 4 | source = "kreuzwerker/docker" 5 | version = "~> 2.13.0" 6 | } 7 | } 8 | } 9 | 10 | provider "docker" {} 11 | 12 | resource "docker_volume" "data" { 13 | count = 2 14 | name = "${var.corral_name}-node-${count.index}" 15 | } 16 | 17 | resource "docker_container" "node" { 18 | count = 2 19 | image = "lscr.io/linuxserver/openssh-server" 20 | name = "${var.corral_name}-node-${count.index}" 21 | 22 | ports { 23 | internal = 2222 24 | } 25 | 26 | env = [ 27 | "PUBLIC_KEY=${var.corral_public_key}", 28 | "USER_NAME=corral", 29 | "USER_PASSWORD=corral", 30 | "SUDO_ACCESS=true", 31 | "PASSWORD_ACCESS=true", 32 | ] 33 | 34 | volumes { 35 | container_path = "/app" 36 | volume_name = docker_volume.data[count.index].name 37 | } 38 | } -------------------------------------------------------------------------------- /examples/multistage/terraform/second/corral.tf: -------------------------------------------------------------------------------- 1 | variable "corral_name" {} 2 | variable "corral_public_key" {} 3 | 4 | output "corral_node_pools" { 5 | value = { 6 | all = [ 7 | for n in docker_container.node : { 8 | name = n.name 9 | user = "corral" 10 | address = "127.0.0.1:${n.ports[0].external}" 11 | } 12 | ] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/multistage/terraform/second/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | docker = { 4 | source = "kreuzwerker/docker" 5 | version = "~> 2.13.0" 6 | } 7 | } 8 | } 9 | 10 | provider "docker" {} 11 | 12 | resource "docker_volume" "data" { 13 | count = 1 14 | name = "${var.corral_name}-node2-${count.index}" 15 | } 16 | 17 | resource "docker_container" "node" { 18 | count = 1 19 | image = "lscr.io/linuxserver/openssh-server" 20 | name = "${var.corral_name}-node2-${count.index}" 21 | 22 | ports { 23 | internal = 2222 24 | } 25 | 26 | env = [ 27 | "PUBLIC_KEY=${var.corral_public_key}", 28 | "USER_NAME=corral", 29 | "USER_PASSWORD=corral", 30 | "SUDO_ACCESS=true", 31 | "PASSWORD_ACCESS=true", 32 | ] 33 | 34 | volumes { 35 | container_path = "/app" 36 | volume_name = docker_volume.data[count.index].name 37 | } 38 | } -------------------------------------------------------------------------------- /examples/overlay-filter/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: overlay-filter 2 | description: > 3 | A simple package for developing corral. 4 | overlay: 5 | a: a 6 | b: b 7 | commands: 8 | - command: echo "corral_set afiles=\"$(ls /app)\"" 9 | node_pools: 10 | - a 11 | - command: echo "corral_set bfiles=\"$(ls /app)\"" 12 | node_pools: 13 | - b 14 | variables: 15 | afiles: 16 | type: string 17 | readOnly: true 18 | description: "Files on a nodes." 19 | bfiles: 20 | type: string 21 | readOnly: true 22 | description: "Files on b nodes." -------------------------------------------------------------------------------- /examples/overlay-filter/overlay/a/app/a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rancherlabs/corral/cab06894ef96a71f7becccee8ebcb2a762a3eeca/examples/overlay-filter/overlay/a/app/a -------------------------------------------------------------------------------- /examples/overlay-filter/overlay/b/app/b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rancherlabs/corral/cab06894ef96a71f7becccee8ebcb2a762a3eeca/examples/overlay-filter/overlay/b/app/b -------------------------------------------------------------------------------- /examples/overlay-filter/overlay/c/app/c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rancherlabs/corral/cab06894ef96a71f7becccee8ebcb2a762a3eeca/examples/overlay-filter/overlay/c/app/c -------------------------------------------------------------------------------- /examples/overlay-filter/terraform/module/corral.tf: -------------------------------------------------------------------------------- 1 | variable "corral_name" {} 2 | variable "corral_public_key" {} 3 | 4 | output "corral_node_pools" { 5 | value = { 6 | a = [ 7 | for n in [docker_container.node[0]] : { 8 | name = n.name 9 | user = "corral" 10 | address = "127.0.0.1:${n.ports[0].external}" 11 | } 12 | ] 13 | b = [ 14 | for n in [docker_container.node[1]] : { 15 | name = n.name 16 | user = "corral" 17 | address = "127.0.0.1:${n.ports[0].external}" 18 | } 19 | ] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /examples/overlay-filter/terraform/module/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | docker = { 4 | source = "kreuzwerker/docker" 5 | version = "~> 2.13.0" 6 | } 7 | } 8 | } 9 | 10 | provider "docker" {} 11 | 12 | resource "docker_volume" "data" { 13 | count = 2 14 | name = "${var.corral_name}-node-${count.index}" 15 | } 16 | 17 | resource "docker_container" "node" { 18 | count = 2 19 | image = "lscr.io/linuxserver/openssh-server" 20 | name = "${var.corral_name}-node-${count.index}" 21 | 22 | ports { 23 | internal = 2222 24 | } 25 | 26 | env = [ 27 | "PUBLIC_KEY=${var.corral_public_key}", 28 | "USER_NAME=corral", 29 | "USER_PASSWORD=corral", 30 | "SUDO_ACCESS=true", 31 | "PASSWORD_ACCESS=true", 32 | ] 33 | 34 | volumes { 35 | container_path = "/app" 36 | volume_name = docker_volume.data[count.index].name 37 | } 38 | } -------------------------------------------------------------------------------- /examples/registry/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: registry 2 | description: | 3 | An authenticated docker registry running in Digitalocean. 4 | variables: 5 | digitalocean_token: 6 | sensitive: true 7 | type: string 8 | optional: false 9 | description: "A Digitalocean API token with write permission. https://docs.digitalocean.com/reference/api/create-personal-access-token/" 10 | digitalocean_domain: 11 | sensitive: true 12 | type: string 13 | optional: false 14 | description: "The domain to use for the registry host." 15 | registry_host: 16 | type: string 17 | readOnly: true 18 | description: "host the configured registry can be accessed at" 19 | username: 20 | type: string 21 | readOnly: true 22 | description: "username for registry authentication" 23 | password: 24 | type: string 25 | readOnly: true 26 | description: "password for registry authentication" 27 | commands: 28 | - module: main 29 | - command: /opt/corral/install.sh 30 | node_pools: 31 | - registry -------------------------------------------------------------------------------- /examples/registry/overlay/etc/docker/registry/config.yml: -------------------------------------------------------------------------------- 1 | version: 0.1 2 | log: 3 | accesslog: 4 | disabled: true 5 | level: info 6 | formatter: text 7 | fields: 8 | service: registry 9 | environment: staging 10 | storage: 11 | filesystem: 12 | rootdirectory: /var/lib/registry 13 | maxthreads: 100 14 | delete: 15 | enabled: false 16 | redirect: 17 | disable: false 18 | auth: 19 | htpasswd: 20 | realm: basic-realm 21 | path: /etc/docker/registry/htpasswd 22 | http: 23 | addr: 0.0.0.0:443 24 | host: https://HOSTNAME 25 | tls: 26 | certificate: /etc/docker/registry/ssl/registry.crt 27 | key: /etc/docker/registry/ssl/registry.key 28 | headers: 29 | X-Content-Type-Options: [nosniff] 30 | http2: 31 | disabled: false 32 | 33 | -------------------------------------------------------------------------------- /examples/registry/overlay/etc/docker/registry/ssl/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rancherlabs/corral/cab06894ef96a71f7becccee8ebcb2a762a3eeca/examples/registry/overlay/etc/docker/registry/ssl/.gitkeep -------------------------------------------------------------------------------- /examples/registry/overlay/etc/systemd/system/registry.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Docker Registry 3 | After=network.target 4 | StartLimitIntervalSec=0 5 | [Service] 6 | Type=simple 7 | Restart=always 8 | RestartSec=1 9 | User=root 10 | ExecStart=/usr/local/bin/registry serve /etc/docker/registry/config.yml 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /examples/registry/overlay/usr/local/bin/registry: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rancherlabs/corral/cab06894ef96a71f7becccee8ebcb2a762a3eeca/examples/registry/overlay/usr/local/bin/registry -------------------------------------------------------------------------------- /examples/registry/terraform/main/corral.tf: -------------------------------------------------------------------------------- 1 | variable "corral_name" {} // name of the corral being created 2 | variable "corral_user_id" {} // how the user is identified (usually github username) 3 | variable "corral_user_public_key" {} // the users public key 4 | variable "corral_public_key" {} // The corrals public key. This should be installed on every node. 5 | variable "corral_private_key" {} // The corrals private key. 6 | 7 | // Package 8 | variable "digitalocean_token" {} 9 | variable "digitalocean_domain" {} 10 | -------------------------------------------------------------------------------- /examples/registry/terraform/main/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13" 3 | required_providers { 4 | digitalocean = { 5 | source = "digitalocean/digitalocean" 6 | version = "~> 2.0" 7 | } 8 | } 9 | } 10 | 11 | provider "random" {} 12 | provider "digitalocean" { 13 | token = var.digitalocean_token 14 | } 15 | 16 | // it is best practice to distinguish an environment with a random id to avoid collisions 17 | resource "random_id" "registry_id" { 18 | byte_length = 6 19 | } 20 | 21 | // we will use the corral public key to get access to nodes to provision them later 22 | resource "digitalocean_ssh_key" "corral_key" { 23 | name = "${var.corral_user_id}-${random_id.registry_id.hex}" 24 | public_key = var.corral_public_key 25 | } 26 | 27 | resource "digitalocean_droplet" "registry" { 28 | count = 1 29 | 30 | name = "${var.corral_user_id}-${random_id.registry_id.hex}-registry" 31 | image = "ubuntu-20-04-x64" 32 | region = "sfo3" 33 | size = "s-1vcpu-2gb" 34 | tags = [var.corral_user_id, random_id.registry_id.hex] // when possible resources should be marked with the associated corral 35 | ssh_keys = [digitalocean_ssh_key.corral_key.id] 36 | } 37 | 38 | resource "digitalocean_record" "registry" { 39 | domain = var.digitalocean_domain 40 | name = random_id.registry_id.hex 41 | type = "A" 42 | value = digitalocean_droplet.registry[0].ipv4_address 43 | } -------------------------------------------------------------------------------- /examples/registry/terraform/main/outputs.tf: -------------------------------------------------------------------------------- 1 | output "corral_node_pools" { 2 | value = { 3 | registry = [ 4 | for droplet in digitalocean_droplet.registry : { 5 | name = droplet.name // unique name of node 6 | user = "root" // ssh username 7 | address = droplet.ipv4_address // address of ssh host 8 | } 9 | ] 10 | } 11 | } 12 | 13 | output "registry_host" { 14 | value = join(".", [digitalocean_record.registry.name, digitalocean_record.registry.domain]) 15 | } -------------------------------------------------------------------------------- /examples/simple/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: simple 2 | description: > 3 | A simple package for developing corral. 4 | commands: 5 | - module: module 6 | - command: /app/setvar1.sh 7 | node_pools: 8 | - all 9 | - command: ls /app 10 | node_pools: 11 | - all 12 | variables: 13 | var1: 14 | type: string 15 | default: "foo" 16 | description: "Example variable for testing variable flows." 17 | var1_out: 18 | type: string 19 | readOnly: true 20 | description: "Set to the value of var1 by a corral set script." -------------------------------------------------------------------------------- /examples/simple/overlay/app/setvar1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | env 4 | 5 | echo "corral_set var1_out=${CORRAL_var1}" 6 | -------------------------------------------------------------------------------- /examples/simple/terraform/module/corral.tf: -------------------------------------------------------------------------------- 1 | variable "corral_name" {} 2 | variable "corral_public_key" {} 3 | 4 | output "corral_node_pools" { 5 | value = { 6 | all = [ 7 | for n in docker_container.node : { 8 | name = n.name 9 | user = "corral" 10 | address = "127.0.0.1:${n.ports[0].external}" 11 | } 12 | ] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/simple/terraform/module/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | docker = { 4 | source = "kreuzwerker/docker" 5 | version = "~> 2.13.0" 6 | } 7 | } 8 | } 9 | 10 | provider "docker" {} 11 | 12 | resource "docker_volume" "data" { 13 | count = 1 14 | name = "${var.corral_name}-node-${count.index}" 15 | } 16 | 17 | resource "docker_container" "node" { 18 | count = 1 19 | image = "lscr.io/linuxserver/openssh-server" 20 | name = "${var.corral_name}-node-${count.index}" 21 | 22 | ports { 23 | internal = 2222 24 | } 25 | 26 | env = [ 27 | "PUBLIC_KEY=${var.corral_public_key}", 28 | "USER_NAME=corral", 29 | "USER_PASSWORD=corral", 30 | "SUDO_ACCESS=true", 31 | "PASSWORD_ACCESS=true", 32 | ] 33 | 34 | volumes { 35 | container_path = "/app" 36 | volume_name = docker_volume.data[count.index].name 37 | } 38 | } -------------------------------------------------------------------------------- /examples/template/README.md: -------------------------------------------------------------------------------- 1 | The package `simple-test` was generated by the following command: 2 | ```shell 3 | /corral package template ../simple test simple-test --description 'A simple template example. This package was generated with the following command: `corral package template -f config.yaml simple-test`' 4 | ``` -------------------------------------------------------------------------------- /examples/template/config.yaml: -------------------------------------------------------------------------------- 1 | name: template 2 | description: > 3 | A simple template example. 4 | This package was generated with the following command: 5 | `corral package template -f config.yaml test simple-test` 6 | packages: 7 | - ../simple -------------------------------------------------------------------------------- /examples/template/simple-test/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: simple-test 2 | description: 'A simple template example. This package was generated with the following command: `corral package template -f config.yaml simple-test`' 3 | commands: 4 | - module: simple/module 5 | - command: /app/setvar1.sh 6 | node_pools: 7 | - all 8 | - command: ls /app 9 | node_pools: 10 | - all 11 | - command: /app/setvar2.sh 12 | node_pools: 13 | - all 14 | variables: 15 | var1: 16 | default: foo 17 | description: Example variable for testing variable flows. 18 | type: string 19 | var1_out: 20 | description: Set to the value of var1 by a corral set script. 21 | readOnly: true 22 | type: string 23 | var2_out: 24 | description: Set to the value of var1_out by a corral set script. 25 | readOnly: true 26 | type: string 27 | -------------------------------------------------------------------------------- /examples/template/simple-test/overlay/app/setvar1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | env 4 | 5 | echo "corral_set var1_out=${CORRAL_var1}" 6 | -------------------------------------------------------------------------------- /examples/template/simple-test/overlay/app/setvar2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | env 4 | 5 | echo "corral_set var2_out=${CORRAL_var1_out}" 6 | -------------------------------------------------------------------------------- /examples/template/simple-test/terraform/simple/module/corral.tf: -------------------------------------------------------------------------------- 1 | variable "corral_name" {} 2 | variable "corral_public_key" {} 3 | 4 | output "corral_node_pools" { 5 | value = { 6 | all = [ 7 | for n in docker_container.node : { 8 | name = n.name 9 | user = "corral" 10 | address = "127.0.0.1:${n.ports[0].external}" 11 | } 12 | ] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/template/simple-test/terraform/simple/module/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | docker = { 4 | source = "kreuzwerker/docker" 5 | version = "~> 2.13.0" 6 | } 7 | } 8 | } 9 | 10 | provider "docker" {} 11 | 12 | resource "docker_volume" "data" { 13 | count = 1 14 | name = "${var.corral_name}-node-${count.index}" 15 | } 16 | 17 | resource "docker_container" "node" { 18 | count = 1 19 | image = "lscr.io/linuxserver/openssh-server" 20 | name = "${var.corral_name}-node-${count.index}" 21 | 22 | ports { 23 | internal = 2222 24 | } 25 | 26 | env = [ 27 | "PUBLIC_KEY=${var.corral_public_key}", 28 | "USER_NAME=corral", 29 | "USER_PASSWORD=corral", 30 | "SUDO_ACCESS=true", 31 | "PASSWORD_ACCESS=true", 32 | ] 33 | 34 | volumes { 35 | container_path = "/app" 36 | volume_name = docker_volume.data[count.index].name 37 | } 38 | } -------------------------------------------------------------------------------- /examples/template/test/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | description: > 3 | A simple package for developing corral. 4 | commands: 5 | - command: /app/setvar2.sh 6 | node_pools: 7 | - all 8 | variables: 9 | var1_out: 10 | type: string 11 | readOnly: true 12 | description: "Set to the value of var1 by a corral set script." 13 | var2_out: 14 | type: string 15 | readOnly: true 16 | description: "Set to the value of var1_out by a corral set script." -------------------------------------------------------------------------------- /examples/template/test/overlay/app/setvar2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | env 4 | 5 | echo "corral_set var2_out=${CORRAL_var1_out}" 6 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rancherlabs/corral 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/blang/semver v3.5.1+incompatible 7 | github.com/containerd/containerd v1.6.2 8 | github.com/hashicorp/go-version v1.4.0 9 | github.com/hashicorp/hc-install v0.3.2 10 | github.com/hashicorp/terraform-exec v0.16.1 11 | github.com/jedib0t/go-pretty/v6 v6.3.0 12 | github.com/magefile/mage v1.13.0 13 | github.com/onsi/ginkgo/v2 v2.1.6 14 | github.com/onsi/gomega v1.20.2 15 | github.com/opencontainers/image-spec v1.0.2 16 | github.com/pkg/errors v0.9.1 17 | github.com/pkg/sftp v1.13.4 18 | github.com/samber/lo v1.27.1 19 | github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 20 | github.com/sirupsen/logrus v1.8.1 21 | github.com/spf13/cobra v1.4.0 22 | github.com/spf13/viper v1.10.1 23 | github.com/stretchr/testify v1.7.1 24 | golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 25 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 26 | gopkg.in/yaml.v3 v3.0.1 27 | gotest.tools/v3 v3.0.3 28 | k8s.io/apimachinery v0.23.5 29 | oras.land/oras-go v1.1.0 30 | ) 31 | 32 | require ( 33 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 34 | github.com/beorn7/perks v1.0.1 // indirect 35 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 36 | github.com/davecgh/go-spew v1.1.1 // indirect 37 | github.com/docker/cli v20.10.14+incompatible // indirect 38 | github.com/docker/distribution v2.8.1+incompatible // indirect 39 | github.com/docker/docker v20.10.14+incompatible // indirect 40 | github.com/docker/docker-credential-helpers v0.6.4 // indirect 41 | github.com/docker/go-connections v0.4.0 // indirect 42 | github.com/docker/go-metrics v0.0.1 // indirect 43 | github.com/docker/go-units v0.4.0 // indirect 44 | github.com/fsnotify/fsnotify v1.5.4 // indirect 45 | github.com/go-logr/logr v1.2.3 // indirect 46 | github.com/gogo/protobuf v1.3.2 // indirect 47 | github.com/golang/protobuf v1.5.2 // indirect 48 | github.com/google/go-cmp v0.5.8 // indirect 49 | github.com/gorilla/mux v1.8.0 // indirect 50 | github.com/hashicorp/errwrap v1.1.0 // indirect 51 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 52 | github.com/hashicorp/go-multierror v1.1.1 // indirect 53 | github.com/hashicorp/hcl v1.0.0 // indirect 54 | github.com/hashicorp/terraform-json v0.13.0 // indirect 55 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 56 | github.com/klauspost/compress v1.15.1 // indirect 57 | github.com/kr/fs v0.1.0 // indirect 58 | github.com/magiconair/properties v1.8.6 // indirect 59 | github.com/mattn/go-runewidth v0.0.13 // indirect 60 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect 61 | github.com/mitchellh/mapstructure v1.5.0 // indirect 62 | github.com/moby/locker v1.0.1 // indirect 63 | github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect 64 | github.com/morikuni/aec v1.0.0 // indirect 65 | github.com/opencontainers/go-digest v1.0.0 // indirect 66 | github.com/pelletier/go-toml v1.9.4 // indirect 67 | github.com/pmezard/go-difflib v1.0.0 // indirect 68 | github.com/prometheus/client_golang v1.12.1 // indirect 69 | github.com/prometheus/client_model v0.2.0 // indirect 70 | github.com/prometheus/common v0.33.0 // indirect 71 | github.com/prometheus/procfs v0.7.3 // indirect 72 | github.com/rivo/uniseg v0.2.0 // indirect 73 | github.com/spf13/afero v1.8.2 // indirect 74 | github.com/spf13/cast v1.4.1 // indirect 75 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 76 | github.com/spf13/pflag v1.0.5 // indirect 77 | github.com/subosito/gotenv v1.2.0 // indirect 78 | github.com/zclconf/go-cty v1.10.0 // indirect 79 | golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect 80 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect 81 | golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 // indirect 82 | golang.org/x/text v0.3.7 // indirect 83 | google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de // indirect 84 | google.golang.org/grpc v1.45.0 // indirect 85 | google.golang.org/protobuf v1.28.0 // indirect 86 | gopkg.in/ini.v1 v1.66.4 // indirect 87 | gopkg.in/yaml.v2 v2.4.0 // indirect 88 | k8s.io/klog/v2 v2.60.1 // indirect 89 | k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect 90 | ) 91 | -------------------------------------------------------------------------------- /magefiles/magefile.go: -------------------------------------------------------------------------------- 1 | //go:build mage 2 | 3 | package main 4 | 5 | import ( 6 | "context" 7 | "log" 8 | "os" 9 | "runtime" 10 | 11 | "github.com/magefile/mage/mg" 12 | "github.com/rancherlabs/corral/magetools" 13 | ) 14 | 15 | var Default = Build 16 | var g *magetools.Go 17 | var version string 18 | var commit string 19 | 20 | func Setup() error { 21 | var err error 22 | version, err = magetools.GetVersion() 23 | if err != nil { 24 | return err 25 | } 26 | commit, err = magetools.GetCommit() 27 | if err != nil { 28 | return err 29 | } 30 | g = magetools.NewGo(goarch(), goos(), version, commit, false, true) 31 | return nil 32 | } 33 | 34 | func Dependencies() error { 35 | mg.Deps(Setup) 36 | return g.Mod().Download() 37 | } 38 | 39 | func Build(ctx context.Context) error { 40 | mg.Deps(Dependencies) 41 | return g.Build() 42 | } 43 | 44 | func Validate() error { 45 | mg.Deps(Setup) 46 | log.Println("[Validate] Running: golangci-lint") 47 | if err := g.Lint(); err != nil { 48 | return err 49 | } 50 | 51 | log.Println("[Validate] Running: go fmt") 52 | if err := g.Fmt("./..."); err != nil { 53 | return err 54 | } 55 | 56 | log.Println("[Validate] Running: go mod tidy") 57 | if err := g.Mod().Tidy(); err != nil { 58 | return err 59 | } 60 | 61 | log.Println("[Validate] Running: go mod verify") 62 | if err := g.Mod().Verify(); err != nil { 63 | return err 64 | } 65 | 66 | log.Println("[Validate] Checking for dirty repo") 67 | if err := magetools.IsGitClean(); err != nil { 68 | return err 69 | } 70 | 71 | log.Println("[Validate] corral has been successfully validated") 72 | return nil 73 | } 74 | 75 | func Test() error { 76 | mg.Deps(Setup) 77 | log.Println("[Test] Running unit tests") 78 | if err := g.Test("", "./cmd/...", "./pkg/..."); err != nil { 79 | return err 80 | } 81 | log.Println("[Test] Running integration tests") 82 | if err := g.Test("./...", "./tests/..."); err != nil { 83 | return err 84 | } 85 | log.Println("[Test] corral has been successfully tested") 86 | return nil 87 | } 88 | 89 | func goos() string { 90 | if goos := os.Getenv("GOOS"); goos != "" { 91 | return goos 92 | } 93 | return runtime.GOOS 94 | } 95 | 96 | func goarch() string { 97 | if goarch := os.Getenv("GOARCH"); goarch != "" { 98 | return goarch 99 | } 100 | return runtime.GOARCH 101 | } 102 | -------------------------------------------------------------------------------- /magetools/magetools.go: -------------------------------------------------------------------------------- 1 | package magetools 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | "github.com/magefile/mage/sh" 10 | ) 11 | 12 | type Go struct { 13 | Arch string 14 | OS string 15 | Version string 16 | Commit string 17 | CGoEnabled string 18 | Verbose string 19 | } 20 | 21 | func NewGo(arch, goos, version, commit string, cgoEnabled, verbose bool) *Go { 22 | return &Go{ 23 | Arch: arch, 24 | OS: goos, 25 | Version: version, 26 | Commit: commit, 27 | CGoEnabled: boolToIntString(cgoEnabled), 28 | Verbose: boolToIntString(verbose), 29 | } 30 | } 31 | 32 | func boolToIntString(b bool) string { 33 | return map[bool]string{true: "1", false: "0"}[b] 34 | } 35 | 36 | func (g *Go) Build() error { 37 | envs := map[string]string{"GOOS": g.OS, "GOARCH": g.Arch, "CGO_ENABLED": g.CGoEnabled, "MAGEFILE_VERBOSE": g.Verbose} 38 | return sh.RunWithV(envs, "go", "build", "-a", "-o", g.output(), "--ldflags="+g.versionFlag(), "main.go") 39 | } 40 | 41 | func (g *Go) output() string { 42 | version := strings.Split(g.Version, "+")[0] 43 | prefix := fmt.Sprintf("dist/corral-%s", version) 44 | if g.OS == "windows" { 45 | return prefix + ".exe" 46 | } else if g.OS == "linux" { 47 | return prefix + "-" + g.Arch 48 | } 49 | return prefix + "-" + g.OS + "-" + g.Arch 50 | } 51 | 52 | func (g *Go) Test(coverpkg string, targets ...string) error { 53 | envs := map[string]string{"GOOS": g.OS, "ARCH": g.Arch, "CGO_ENABLED": g.CGoEnabled, "MAGEFILE_VERBOSE": g.Verbose} 54 | if coverpkg != "" { 55 | coverpkg = "-coverpkg=" + coverpkg 56 | } 57 | return sh.RunWithV(envs, "go", append([]string{"test", "-v", "-cover", coverpkg, "--ldflags=" + g.versionFlag()}, targets...)...) 58 | } 59 | 60 | type Mod struct { 61 | *Go 62 | } 63 | 64 | func (g *Go) Mod() *Mod { 65 | return &Mod{g} 66 | } 67 | 68 | func (m *Mod) Download() error { 69 | envs := map[string]string{"GOOS": m.OS, "ARCH": m.Arch} 70 | return sh.RunWithV(envs, "go", "mod", "download") 71 | } 72 | 73 | func (m *Mod) Tidy() error { 74 | envs := map[string]string{"GOOS": m.OS, "ARCH": m.Arch} 75 | return sh.RunWithV(envs, "go", "mod", "tidy") 76 | } 77 | 78 | func (m *Mod) Verify() error { 79 | envs := map[string]string{"GOOS": m.OS, "ARCH": m.Arch} 80 | return sh.RunWithV(envs, "go", "mod", "verify") 81 | } 82 | 83 | func (g *Go) Fmt(target string) error { 84 | envs := map[string]string{"GOOS": g.OS, "ARCH": g.Arch} 85 | return sh.RunWithV(envs, "go", "fmt", target) 86 | } 87 | 88 | func (g *Go) Lint(targets ...string) error { 89 | envs := map[string]string{"GOOS": g.OS, "ARCH": g.Arch, "CGO_ENABLED": g.CGoEnabled, "MAGEFILE_VERBOSE": g.Verbose} 90 | return sh.RunWithV(envs, "golangci-lint", append([]string{"run"}, targets...)...) 91 | } 92 | 93 | func (g *Go) versionFlag() string { 94 | return fmt.Sprintf(`-X 'github.com/rancherlabs/corral/pkg/version.Version=%s'`, g.Version) 95 | } 96 | 97 | var ErrDirtyRepo = errors.New("encountered dirty repo") 98 | 99 | func GetCommit() (string, error) { 100 | result, err := sh.Output("git", "rev-parse", "--short", "HEAD") 101 | if err != nil { 102 | return "", err 103 | } 104 | if err = IsGitClean(); err != nil { 105 | if !errors.Is(err, ErrDirtyRepo) { 106 | return "", err 107 | } 108 | result += "-dirty" 109 | } 110 | return strings.TrimSpace(result), nil 111 | } 112 | 113 | func IsGitClean() error { 114 | result, err := sh.Output("git", "status", "--porcelain", "--untracked-files=no") 115 | if err != nil { 116 | return err 117 | } 118 | if result != "" { 119 | return ErrDirtyRepo 120 | } 121 | return nil 122 | } 123 | 124 | func GetVersion() (string, error) { 125 | commit, err := GetCommit() 126 | if err != nil { 127 | return "", err 128 | } 129 | ref := os.Getenv("GITHUB_REF_NAME") 130 | if ref != "" { 131 | ref = strings.TrimPrefix(ref, "v") + "+" + commit // append build metadata 132 | } else { 133 | ref = commit 134 | } 135 | return ref, nil 136 | } 137 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/rancherlabs/corral/cmd" 7 | "github.com/rancherlabs/corral/pkg/config" 8 | "github.com/samber/lo" 9 | ) 10 | 11 | func main() { 12 | root := os.Getenv("CORRAL_ROOT") 13 | if root == "" { 14 | root = lo.Must(os.UserHomeDir()) 15 | } 16 | config.InitializeRootPath(root) 17 | cmd.Execute() 18 | } 19 | -------------------------------------------------------------------------------- /pkg/cmd/output.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | 7 | "github.com/jedib0t/go-pretty/v6/table" 8 | "github.com/sirupsen/logrus" 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | type OutputFormat string 13 | 14 | const ( 15 | OutputFormatTable OutputFormat = "table" 16 | OutputFormatJSON OutputFormat = "json" 17 | OutputFormatYAML OutputFormat = "yaml" 18 | ) 19 | 20 | func (e *OutputFormat) String() string { 21 | return string(*e) 22 | } 23 | 24 | var ErrUnknownOutputFormat = errors.New(`must be one of "table", "json", or "yaml"`) 25 | 26 | // Set must have pointer receiver so it doesn't change the value of a copy 27 | func (e *OutputFormat) Set(v string) error { 28 | switch v { 29 | case "table", "json", "yaml": 30 | *e = OutputFormat(v) 31 | return nil 32 | default: 33 | return ErrUnknownOutputFormat 34 | } 35 | } 36 | 37 | func (e *OutputFormat) Type() string { 38 | return "" 39 | } 40 | 41 | type OutputOptions struct { 42 | Key string 43 | Value string 44 | } 45 | 46 | func Output[K comparable, V any](out map[K]V, output OutputFormat, opts OutputOptions) (string, error) { 47 | switch output { 48 | case OutputFormatTable: 49 | tbl := table.NewWriter() 50 | tbl.AppendHeader(table.Row{opts.Key, opts.Value}) 51 | tbl.AppendSeparator() 52 | for k, v := range out { 53 | tbl.AppendRow(table.Row{k, v}) 54 | } 55 | return tbl.Render(), nil 56 | case OutputFormatJSON: 57 | data, err := json.Marshal(&out) 58 | if err != nil { 59 | return "", err 60 | } 61 | return string(data), nil 62 | case OutputFormatYAML: 63 | data, err := yaml.Marshal(&out) 64 | if err != nil { 65 | logrus.Error(err) 66 | return "", err 67 | } 68 | return string(data[:len(data)-1]), nil // remove trailing newline 69 | } 70 | return "", ErrUnknownOutputFormat 71 | } 72 | -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "os" 7 | 8 | tfversion "github.com/hashicorp/go-version" 9 | install "github.com/hashicorp/hc-install" 10 | "github.com/hashicorp/hc-install/fs" 11 | "github.com/hashicorp/hc-install/product" 12 | "github.com/hashicorp/hc-install/releases" 13 | "github.com/hashicorp/hc-install/src" 14 | "github.com/hashicorp/terraform-exec/tfexec" 15 | "github.com/rancherlabs/corral/pkg/version" 16 | "github.com/sirupsen/logrus" 17 | "gopkg.in/yaml.v3" 18 | ) 19 | 20 | type Config struct { 21 | UserID string `yaml:"user_id"` 22 | UserPublicKeyPath string `yaml:"user_public_key_path"` 23 | 24 | Version string `yaml:"version"` 25 | 26 | Vars map[string]any `yaml:"vars"` 27 | } 28 | 29 | func MustLoad() Config { 30 | c, err := Load() 31 | if err != nil { 32 | if errors.Is(err, os.ErrNotExist) { 33 | logrus.Fatal("You must call `corral config` before using this command.") 34 | } 35 | logrus.Fatal("Configuration file is invalid", err) 36 | } 37 | return c 38 | } 39 | 40 | func Load() (Config, error) { 41 | var c Config 42 | 43 | body, err := os.ReadFile(CorralRoot("config.yaml")) 44 | if err != nil { 45 | return Config{}, err 46 | } 47 | 48 | if err = yaml.Unmarshal(body, &c); err != nil { 49 | return Config{}, err 50 | } 51 | 52 | if version.Version != c.Version { 53 | logrus.Infof("Upgrading corral to %s.", version.Version) 54 | if err = Install(); err != nil { 55 | panic(err) 56 | } 57 | 58 | c.Version = version.Version 59 | if err = c.Save(); err != nil { 60 | panic(err) 61 | } 62 | } 63 | 64 | return c, nil 65 | } 66 | 67 | func (c *Config) Save() (err error) { 68 | f, err := os.Create(CorralRoot("config.yaml")) 69 | defer func(f *os.File) { _ = f.Close() }(f) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | return yaml.NewEncoder(f).Encode(c) 75 | } 76 | 77 | func NewTerraform(modulePath, version string) (*tfexec.Terraform, error) { 78 | v, err := tfversion.NewVersion(version) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | versionPath := CorralRoot("cache", "terraform", "bin", version) 84 | 85 | if err = os.MkdirAll(versionPath, 0o700); err != nil { 86 | return nil, err 87 | } 88 | 89 | i := install.NewInstaller() 90 | tfPath, err := i.Ensure(context.Background(), []src.Source{ 91 | &fs.ExactVersion{ 92 | Product: product.Terraform, 93 | ExtraPaths: []string{versionPath}, 94 | Version: v, 95 | }, 96 | &releases.ExactVersion{ 97 | Product: product.Terraform, 98 | InstallDir: versionPath, 99 | Version: v, 100 | }, 101 | }) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | tf, err := tfexec.NewTerraform(modulePath, tfPath) 107 | 108 | if logrus.GetLevel() == logrus.DebugLevel { 109 | tf.SetStdout(os.Stdout) 110 | tf.SetStderr(os.Stderr) 111 | } 112 | 113 | return tf, err 114 | } 115 | -------------------------------------------------------------------------------- /pkg/config/install.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | func Install() error { 8 | initialPaths := []string{ 9 | CorralRoot("cache", "layers"), 10 | CorralRoot("cache", "packages"), 11 | CorralRoot("cache", "terraform", "bin"), 12 | } 13 | 14 | for _, p := range initialPaths { 15 | if err := os.MkdirAll(p, 0o700); err != nil { 16 | return err 17 | } 18 | } 19 | 20 | return nil 21 | } 22 | -------------------------------------------------------------------------------- /pkg/config/install_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestInstall(t *testing.T) { 10 | rootPath = t.TempDir() 11 | 12 | require.NoError(t, Install()) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/config/paths.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "path/filepath" 5 | ) 6 | 7 | func CorralRoot(parts ...string) string { 8 | return filepath.Join(append([]string{rootPath, ".corral"}, parts...)...) 9 | } 10 | 11 | func CorralPath(name string) string { 12 | return CorralRoot("corrals", name) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/config/root_path.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/samber/lo" 7 | ) 8 | 9 | var rootPath string 10 | 11 | func InitializeRootPath(path string) { 12 | rootPath = path 13 | lo.Must(os.Stat(rootPath)) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/corral/corral.go: -------------------------------------------------------------------------------- 1 | package corral 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/hashicorp/terraform-exec/tfexec" 11 | "github.com/rancherlabs/corral/pkg/config" 12 | "github.com/rancherlabs/corral/pkg/version" 13 | 14 | "github.com/pkg/errors" 15 | 16 | "github.com/rancherlabs/corral/pkg/vars" 17 | 18 | "github.com/sirupsen/logrus" 19 | "gopkg.in/yaml.v3" 20 | ) 21 | 22 | const nodePoolVarName = "corral_node_pools" 23 | 24 | type Corral struct { 25 | RootPath string `yaml:"rootPath"` 26 | Source string `yaml:"source"` 27 | 28 | Name string `yaml:"name"` 29 | Status Status `yaml:"status" json:"status,omitempty"` 30 | PublicKey string `yaml:"public_key"` 31 | PrivateKey string `yaml:"private_key"` 32 | 33 | NodePools map[string][]Node `yaml:"node_pools" json:"node_pools,omitempty"` 34 | Vars vars.VarSet `yaml:"vars" json:"vars,omitempty"` 35 | } 36 | 37 | func Load(path string) (*Corral, error) { 38 | var c Corral 39 | b, err := os.ReadFile(filepath.Join(path, "corral.yaml")) 40 | if err != nil { 41 | return nil, err 42 | } 43 | return &c, yaml.Unmarshal(b, &c) 44 | } 45 | 46 | func (c *Corral) TerraformPath(name string) string { 47 | return filepath.Join(c.RootPath, "terraform", name) 48 | } 49 | 50 | func (c *Corral) Exists() bool { 51 | _, err := os.Stat(c.RootPath) 52 | return !errors.Is(err, os.ErrNotExist) 53 | } 54 | 55 | func (c *Corral) Save() error { 56 | err := os.MkdirAll(c.RootPath, 0700) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | err = os.MkdirAll(c.TerraformPath(""), 0700) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | f, err := os.Create(filepath.Join(c.RootPath, "corral.yaml")) 67 | defer func(f *os.File) { _ = f.Close() }(f) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | return yaml.NewEncoder(f).Encode(c) 73 | } 74 | 75 | func (c *Corral) Delete() error { 76 | err := os.RemoveAll(c.RootPath) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | return nil 82 | } 83 | 84 | func (c *Corral) SetStatus(status Status) { 85 | c.Status = status 86 | err := c.Save() 87 | if err != nil { 88 | logrus.Warn("Failed to save corral status") 89 | } 90 | } 91 | 92 | func (c *Corral) ApplyModule(src, name string) error { 93 | if err := os.MkdirAll(c.TerraformPath(name), 0700); err != nil { 94 | return err 95 | } 96 | 97 | tf, err := config.NewTerraform(c.TerraformPath(name), version.TerraformVersion) 98 | if err != nil { 99 | return errors.Wrap(err, "failed to initialize terraform") 100 | } 101 | 102 | err = tf.Init(context.Background(), 103 | tfexec.Upgrade(false), 104 | tfexec.FromModule(src)) 105 | if err != nil { 106 | return errors.Wrap(err, "failed to initialize terraform module") 107 | } 108 | 109 | f, err := os.Create(filepath.Join(c.TerraformPath(name), "terraform.tfvars.json")) 110 | if err != nil { 111 | return errors.Wrap(err, "failed to create tfvars file") 112 | } 113 | tfVars := map[string]any{} 114 | for k, v := range c.Vars { 115 | if k == "corral_node_pools" { 116 | tfVars[k] = c.NodePools 117 | } 118 | 119 | tfVars[k] = v 120 | } 121 | 122 | _ = json.NewEncoder(f).Encode(tfVars) 123 | _ = f.Close() 124 | 125 | err = tf.Apply(context.Background()) 126 | if err != nil { 127 | return errors.Wrap(err, "failed to apply terraform module") 128 | } 129 | 130 | tfOutput, err := tf.Output(context.Background()) 131 | if err != nil { 132 | return errors.Wrap(err, "failed read terraform output") 133 | } 134 | 135 | for k, v := range tfOutput { 136 | if k == nodePoolVarName { 137 | np := map[string][]Node{} 138 | if err = json.Unmarshal(v.Value, &np); err != nil { 139 | return errors.Wrap(err, "failed to parse node pools from output") 140 | } 141 | 142 | for s, nodes := range np { 143 | c.NodePools[s] = append(c.NodePools[s], nodes...) 144 | } 145 | 146 | var buf bytes.Buffer 147 | _ = json.NewEncoder(&buf).Encode(c.NodePools) 148 | c.Vars[nodePoolVarName] = vars.Escape(&buf) 149 | } 150 | 151 | val, err := vars.FromTerraformOutputMeta(v) 152 | if err != nil { 153 | return errors.Wrap(err, "failed to parse variable terraform output") 154 | } 155 | c.Vars[k] = val 156 | } 157 | 158 | return nil 159 | } 160 | 161 | func (c *Corral) DestroyModule(name string) error { 162 | if _, err := os.Stat(c.TerraformPath(name)); os.IsNotExist(err) { 163 | return nil 164 | } 165 | 166 | tf, err := config.NewTerraform(c.TerraformPath(name), version.TerraformVersion) 167 | if err != nil { 168 | return errors.Wrap(err, "failed to initialized terraform") 169 | } 170 | 171 | err = tf.Destroy(context.Background()) 172 | if err != nil { 173 | return errors.Wrap(err, "failed to destroy terraform module") 174 | } 175 | 176 | return nil 177 | } 178 | -------------------------------------------------------------------------------- /pkg/corral/node.go: -------------------------------------------------------------------------------- 1 | package corral 2 | 3 | type Node struct { 4 | Name string `json:"name,omitempty" yaml:"name,omitempty"` 5 | User string `json:"user,omitempty" yaml:"user,omitempty"` 6 | Address string `json:"address,omitempty" yaml:"address,omitempty"` 7 | BastionAddress string `json:"bastion_address,omitempty" yaml:"bastion_address,omitempty"` 8 | OverlayRoot string `json:"overlay_root" yaml:"overlay_root"` 9 | } 10 | -------------------------------------------------------------------------------- /pkg/corral/status.go: -------------------------------------------------------------------------------- 1 | package corral 2 | 3 | type Status int 4 | 5 | const ( 6 | StatusNew = 0 + iota 7 | StatusProvisioning 8 | StatusError 9 | StatusDeleting 10 | StatusReady 11 | ) 12 | 13 | var statusStringMap = map[Status]string{ 14 | StatusNew: "NEW", 15 | StatusProvisioning: "PROVISIONING", 16 | StatusError: "ERROR", 17 | StatusDeleting: "DELETING", 18 | StatusReady: "READY", 19 | } 20 | 21 | func (s Status) String() string { 22 | return statusStringMap[s] 23 | } 24 | -------------------------------------------------------------------------------- /pkg/package/fetcher.go: -------------------------------------------------------------------------------- 1 | package _package 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "io/fs" 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/containerd/containerd/remotes" 12 | v1 "github.com/opencontainers/image-spec/specs-go/v1" 13 | "github.com/rancherlabs/corral/pkg/config" 14 | ) 15 | 16 | type CachedFetcher struct { 17 | cachePath string 18 | source remotes.Fetcher 19 | } 20 | 21 | func NewCachedFetcher(source remotes.Fetcher) remotes.Fetcher { 22 | return &CachedFetcher{ 23 | cachePath: config.CorralRoot("cache", "layers"), 24 | source: source, 25 | } 26 | } 27 | 28 | func (c *CachedFetcher) Fetch(ctx context.Context, desc v1.Descriptor) (io.ReadCloser, error) { 29 | isCached, err := c.isCached(desc) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | if !isCached { 35 | if err := c.cacheFromSource(ctx, desc); err != nil { 36 | return nil, err 37 | } 38 | } 39 | 40 | f, err := os.Open(c.descriptorPath(desc)) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | return f, nil 46 | } 47 | 48 | func (c *CachedFetcher) isCached(desc v1.Descriptor) (bool, error) { 49 | info, err := os.Stat(c.descriptorPath(desc)) 50 | if errors.Is(err, fs.ErrNotExist) { 51 | return false, nil 52 | } 53 | if err != nil { 54 | return false, err 55 | } 56 | 57 | return info.Size() == desc.Size, nil 58 | } 59 | 60 | func (c *CachedFetcher) cacheFromSource(ctx context.Context, desc v1.Descriptor) error { 61 | // download the image from the source 62 | r, err := c.source.Fetch(ctx, desc) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | defer func() { _ = r.Close() }() 68 | 69 | // create the cache file 70 | f, err := os.Create(c.descriptorPath(desc)) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | // copy the downloaded layer to the cache 76 | _, err = io.Copy(f, r) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | return nil 82 | } 83 | 84 | func (c *CachedFetcher) descriptorPath(desc v1.Descriptor) string { 85 | return filepath.Join(c.cachePath, desc.Digest.String()) 86 | } 87 | -------------------------------------------------------------------------------- /pkg/package/load.go: -------------------------------------------------------------------------------- 1 | package _package 2 | 3 | import ( 4 | "archive/tar" 5 | "compress/gzip" 6 | "context" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "io" 11 | "io/fs" 12 | "os" 13 | "path/filepath" 14 | "strings" 15 | 16 | "github.com/blang/semver" 17 | "github.com/rancherlabs/corral/pkg/version" 18 | 19 | "github.com/opencontainers/image-spec/specs-go/v1" 20 | "github.com/rancherlabs/corral/pkg/config" 21 | "github.com/sirupsen/logrus" 22 | ) 23 | 24 | func LoadPackage(ref string) (Package, error) { 25 | path, _ := filepath.Abs(ref) 26 | if _, err := os.Stat(path); err == nil { 27 | return loadLocalPackage(path) 28 | } else if strings.HasPrefix(ref, "./") { 29 | return Package{}, err 30 | } 31 | 32 | if !strings.Contains(ref, ":") { 33 | logrus.Info("Defaulting to latest tag.") 34 | ref += ":latest" 35 | } 36 | 37 | return loadRemotePackage(ref) 38 | } 39 | 40 | func loadLocalPackage(src string) (pkg Package, err error) { 41 | if _, err = os.Stat(src); err != nil { 42 | return pkg, err 43 | } 44 | 45 | pkg.RootPath = src 46 | 47 | pkg.Manifest, err = LoadManifest(os.DirFS(pkg.RootPath), "manifest.yaml") 48 | if err != nil { 49 | return pkg, err 50 | } 51 | 52 | return pkg, nil 53 | } 54 | 55 | func loadRemotePackage(ref string) (pkg Package, err error) { 56 | registryStore, err := newRegistryStore() 57 | if err != nil { 58 | return 59 | } 60 | 61 | // get the latest digest for the ref 62 | _, desc, err := registryStore.Resolve(context.Background(), ref) 63 | if err != nil { 64 | return 65 | } 66 | 67 | // check if this digest has already been cached 68 | dest := filepath.Join(config.CorralRoot("cache", "packages"), getRefPath(ref, string(desc.Digest))) 69 | _, err = os.Stat(dest) 70 | if err == nil { 71 | return loadLocalPackage(dest) 72 | } 73 | if !errors.Is(err, os.ErrNotExist) { 74 | return 75 | } 76 | 77 | // use a cached fetcher to cache the layers 78 | storeFetcher, err := registryStore.Fetcher(context.Background(), ref) 79 | if err != nil { 80 | return 81 | } 82 | 83 | fetcher := NewCachedFetcher(storeFetcher) 84 | if err != nil { 85 | return 86 | } 87 | 88 | // fetch the image manifest 89 | r, err := fetcher.Fetch(context.Background(), desc) 90 | if err != nil { 91 | return 92 | } 93 | 94 | var manifest v1.Manifest 95 | err = json.NewDecoder(r).Decode(&manifest) 96 | if err != nil { 97 | return 98 | } 99 | 100 | if ver, err := semver.Parse(manifest.Annotations[CorralVersionAnnotation]); err == nil { 101 | if semver.MustParse(version.Version).Major > ver.Major { 102 | return pkg, errors.New("packages must be published by the same major version of corral or later") 103 | } 104 | } else { 105 | logrus.Warn("package does not have valid corral version annotation") 106 | } 107 | 108 | // create the destination directory 109 | err = os.MkdirAll(dest, 0o700) 110 | if err != nil { 111 | return 112 | } 113 | 114 | // extract the layers to the destination 115 | for _, layer := range manifest.Layers { 116 | var layerReader io.ReadCloser 117 | layerReader, err = fetcher.Fetch(context.Background(), layer) 118 | if err != nil { 119 | return 120 | } 121 | 122 | if err = extractLayer(dest, layerReader); err != nil { 123 | return 124 | } 125 | } 126 | 127 | pkg, err = loadLocalPackage(dest) 128 | if err != nil { 129 | return pkg, err 130 | } 131 | 132 | if pkg.Annotations[CorralVersionAnnotation] != "" { 133 | pkv, err := semver.Parse(pkg.Annotations[CorralVersionAnnotation]) 134 | if err != nil { 135 | logrus.Warningf("package has invalid version annotation: %s", pkg.Annotations[CorralVersionAnnotation]) 136 | return pkg, nil 137 | } 138 | 139 | if semver.MustParseRange("< 0.2.0-alpha0")(pkv) { 140 | err = migrateScriptsToOverlay(pkg) 141 | if err != nil { 142 | return pkg, fmt.Errorf("failed to migrate scripts to overlay format: %w", err) 143 | } 144 | } 145 | } 146 | 147 | return pkg, nil 148 | } 149 | 150 | func extractLayer(dest string, r io.Reader) error { 151 | gr, err := gzip.NewReader(r) 152 | if err != nil { 153 | return err 154 | } 155 | 156 | tr := tar.NewReader(gr) 157 | for { 158 | header, err := tr.Next() 159 | if err == io.EOF { 160 | break // End of archive 161 | } 162 | 163 | if header.Typeflag == tar.TypeDir { 164 | err = os.MkdirAll(filepath.Join(dest, header.Name), 0o700) 165 | if err != nil { 166 | return err 167 | } 168 | continue 169 | } 170 | 171 | destPath, _ := filepath.Split(header.Name) 172 | err = os.MkdirAll(filepath.Join(dest, destPath), 0o700) 173 | if err != nil { 174 | return err 175 | } 176 | 177 | f, err := os.Create(filepath.Join(dest, header.Name)) 178 | if err != nil { 179 | return err 180 | } 181 | 182 | _, err = io.Copy(f, tr) 183 | if err != nil { 184 | _ = f.Close() 185 | return err 186 | } 187 | 188 | _ = f.Close() 189 | } 190 | 191 | return nil 192 | } 193 | 194 | func getRefPath(ref, digest string) string { 195 | ref = strings.Split(ref, ":")[0] 196 | digest = strings.Split(digest, ":")[1] 197 | 198 | return filepath.Join(append(strings.Split(ref, "/"), digest)...) 199 | } 200 | 201 | func migrateScriptsToOverlay(pkg Package) error { 202 | scriptsPath := filepath.Join(pkg.RootPath, "scripts") 203 | overlayPath := pkg.OverlayPath() + "/opt/corral" 204 | 205 | return filepath.WalkDir(scriptsPath, func(path string, d fs.DirEntry, err error) error { 206 | if err != nil { 207 | return err 208 | } 209 | 210 | dest := overlayPath + path[len(scriptsPath):] 211 | 212 | if d.IsDir() { 213 | err = os.Mkdir(dest, 0x700) 214 | if err != nil { 215 | return err 216 | } 217 | 218 | return nil 219 | } 220 | 221 | var srcFile *os.File 222 | var destFile *os.File 223 | srcFile, err = os.Open(path) 224 | if err != nil { 225 | return err 226 | } 227 | 228 | destFile, err = os.Create(dest) 229 | if err != nil { 230 | return err 231 | } 232 | 233 | _, err = io.Copy(destFile, srcFile) 234 | 235 | _ = srcFile.Close() 236 | _ = destFile.Close() 237 | 238 | return err 239 | }) 240 | } 241 | -------------------------------------------------------------------------------- /pkg/package/manifest.go: -------------------------------------------------------------------------------- 1 | package _package 2 | 3 | import ( 4 | "bytes" 5 | _ "embed" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "io/fs" 10 | 11 | "github.com/pkg/errors" 12 | "github.com/rancherlabs/corral/pkg/vars" 13 | "github.com/santhosh-tekuri/jsonschema/v5" 14 | "gopkg.in/yaml.v3" 15 | ) 16 | 17 | type Command struct { 18 | // shell fields 19 | Command string `yaml:"command,omitempty"` 20 | NodePoolNames []string `yaml:"node_pools,omitempty"` 21 | Parallel *bool `yaml:"parallel,omitempty"` 22 | 23 | // terraform module fields 24 | Module string `yaml:"module,omitempty"` 25 | SkipCleanup bool `yaml:"skip_cleanup,omitempty"` 26 | } 27 | 28 | type VariableSchemas map[string]Schema 29 | 30 | type Manifest struct { 31 | Name string `yaml:"name"` 32 | Annotations map[string]string `yaml:"annotations,omitempty"` 33 | Description string `yaml:"description,omitempty"` 34 | Commands []Command `yaml:"commands"` 35 | Overlay map[string]string `yaml:"overlay,omitempty"` 36 | VariableSchemas VariableSchemas `yaml:"variables,omitempty"` 37 | } 38 | 39 | //go:embed package-manifest.schema.json 40 | var manifestSchemaBytes []byte 41 | 42 | var manifestSchema *jsonschema.Schema 43 | var schemaCompiler *jsonschema.Compiler 44 | 45 | func init() { 46 | schemaCompiler = jsonschema.NewCompiler() 47 | schemaCompiler.Draft = jsonschema.Draft7 48 | 49 | _ = schemaCompiler.AddResource("manifest", bytes.NewReader(manifestSchemaBytes)) 50 | manifestSchema = schemaCompiler.MustCompile("manifest") 51 | manifestSchema.Location = "Package Manifest" 52 | } 53 | 54 | // LoadManifest reads a manifest file and validates it is a valid manifest. 55 | func LoadManifest(_fs fs.FS, path string) (Manifest, error) { 56 | var manifest Manifest 57 | 58 | f, err := _fs.Open(path) 59 | if err != nil { 60 | return manifest, err 61 | } 62 | defer func(f fs.File) { _ = f.Close() }(f) 63 | 64 | buf, err := io.ReadAll(f) 65 | if err != nil { 66 | return manifest, err 67 | } 68 | _ = f.Close() 69 | 70 | err = ValidateManifest(buf) 71 | if err != nil { 72 | return manifest, err 73 | } 74 | 75 | err = yaml.Unmarshal(buf, &manifest) 76 | if err != nil { 77 | return manifest, err 78 | } 79 | 80 | if manifest.Annotations == nil { 81 | manifest.Annotations = map[string]string{} 82 | } 83 | 84 | return manifest, err 85 | } 86 | 87 | func (m *Manifest) ApplyDefaultVars(vs vars.VarSet) error { 88 | for k, schema := range m.VariableSchemas { 89 | if _, ok := vs[k]; !ok { 90 | if schema.Default == nil { 91 | continue 92 | } 93 | value := schema.Default 94 | vs[k] = value 95 | } 96 | } 97 | 98 | return nil 99 | } 100 | 101 | // ValidateDefaults returns an error if the var set does not match the manifest variable schemas. 102 | func (m *Manifest) ValidateDefaults() error { 103 | for _, schema := range m.VariableSchemas { 104 | if schema.Default != nil { 105 | err := schema.Validate(schema.Default) 106 | if err != nil { 107 | return err 108 | } 109 | } 110 | } 111 | 112 | return nil 113 | } 114 | 115 | // ValidateVarSet returns an error if the var set does not match the manifest variable schemas. 116 | func (m *Manifest) ValidateVarSet(vs vars.VarSet, write bool) error { 117 | for k, schema := range m.VariableSchemas { 118 | 119 | if _, ok := vs[k]; schema.ReadOnly && write && ok { 120 | return fmt.Errorf("[%s] may not be set", k) 121 | } 122 | 123 | if _, ok := vs[k]; !schema.Optional && !schema.ReadOnly && schema.Default == nil && !ok { 124 | return fmt.Errorf("[%s] is required", k) 125 | } 126 | 127 | if err := schema.Validate(vs[k]); err != nil && vs[k] != nil { 128 | return errors.Wrap(err, k+":") 129 | } 130 | } 131 | 132 | return nil 133 | } 134 | 135 | // FilterVars returns the given VarSet without any variables not defined in the manifest 136 | func (m *Manifest) FilterVars(vs vars.VarSet) vars.VarSet { 137 | rval := vars.VarSet{} 138 | 139 | for k, v := range vs { 140 | if _, ok := m.VariableSchemas[k]; !ok { 141 | continue 142 | } 143 | 144 | rval[k] = v 145 | } 146 | 147 | return rval 148 | } 149 | 150 | // FilterSensitiveVars returns the given VarSet without any variables marked as sensitive in the manifest 151 | func (m *Manifest) FilterSensitiveVars(vs vars.VarSet) vars.VarSet { 152 | rval := vars.VarSet{} 153 | 154 | for k, v := range vs { 155 | schema := m.VariableSchemas[k] 156 | 157 | if schema.Sensitive { 158 | continue 159 | } 160 | 161 | rval[k] = v 162 | } 163 | 164 | return rval 165 | } 166 | 167 | func (m *Manifest) GetAnnotation(key string) string { 168 | if m.Annotations != nil { 169 | return m.Annotations[key] 170 | } 171 | 172 | return "" 173 | } 174 | 175 | // ValidateManifest returns an error of the manifest violates any rules defined in the package-manifest.schema.json 176 | func ValidateManifest(manifest []byte) error { 177 | var yml any 178 | 179 | _ = yaml.Unmarshal(manifest, &yml) 180 | 181 | j := toStringKeys(yml) 182 | 183 | return manifestSchema.Validate(j) 184 | } 185 | 186 | // toStringKeys converts any map[any]any to map[string]any recursively. 187 | func toStringKeys(val any) any { 188 | switch val := val.(type) { 189 | case map[any]any: 190 | m := make(map[string]any) 191 | for k, v := range val { 192 | k := k.(string) 193 | m[k] = toStringKeys(v) 194 | } 195 | return m 196 | case []any: 197 | var l = make([]any, len(val)) 198 | for i, v := range val { 199 | l[i] = toStringKeys(v) 200 | } 201 | return l 202 | default: 203 | return val 204 | } 205 | } 206 | 207 | func (s *VariableSchemas) UnmarshalYAML(unmarshal func(any) error) error { 208 | *s = VariableSchemas{} 209 | _vars := struct { 210 | VariableSchemas map[string]any `yaml:"variables,inline,omitempty"` 211 | }{ 212 | VariableSchemas: map[string]any{}, 213 | } 214 | err := unmarshal(&_vars) 215 | if err != nil { 216 | return err 217 | } 218 | 219 | for k, v := range _vars.VariableSchemas { 220 | var buf bytes.Buffer 221 | var schema Schema 222 | 223 | _ = json.NewEncoder(&buf).Encode(toStringKeys(v)) 224 | _ = schemaCompiler.AddResource(k, &buf) 225 | schema.Schema = schemaCompiler.MustCompile(k) 226 | schema.Location = k 227 | 228 | if vv, ok := v.(map[string]any); ok { 229 | if val, okk := vv["sensitive"].(bool); okk && val { 230 | schema.Sensitive = true 231 | } 232 | 233 | if val, okk := vv["optional"].(bool); okk && val { 234 | schema.Optional = true 235 | } 236 | 237 | if val, okk := vv["readOnly"].(bool); okk && val { 238 | schema.ReadOnly = true 239 | } 240 | 241 | if description, okk := vv["description"].(string); okk { 242 | schema.Description = description 243 | } 244 | 245 | if dflt, okk := vv["default"]; okk { 246 | schema.Default = dflt 247 | } 248 | } 249 | 250 | (*s)[k] = schema 251 | } 252 | 253 | return nil 254 | } 255 | -------------------------------------------------------------------------------- /pkg/package/manifest_test.go: -------------------------------------------------------------------------------- 1 | package _package_test 2 | 3 | import ( 4 | "embed" 5 | "testing" 6 | 7 | _package "github.com/rancherlabs/corral/pkg/package" 8 | "github.com/rancherlabs/corral/pkg/vars" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | //go:embed tests 13 | var _fs embed.FS 14 | 15 | func TestLoadManifest(t *testing.T) { 16 | { // valid 17 | res, err := _package.LoadManifest(_fs, "tests/valid.yaml") 18 | 19 | assert.NoError(t, err) 20 | 21 | assert.Equal(t, "valid", res.Name) 22 | assert.Equal(t, "valid description", res.Description) 23 | 24 | assert.NotNil(t, res.Annotations) 25 | assert.Equal(t, res.Annotations["foo"], "bar") 26 | assert.Equal(t, res.Annotations["baz"], "1") 27 | 28 | assert.NotNil(t, res.Overlay) 29 | assert.Equal(t, res.Annotations["foo"], "bar") 30 | 31 | assert.Len(t, res.Commands, 2) 32 | assert.Equal(t, "module", res.Commands[0].Module) 33 | assert.True(t, res.Commands[0].SkipCleanup) 34 | 35 | assert.Len(t, res.Commands[1].NodePoolNames, 2) 36 | assert.Equal(t, "foo", res.Commands[1].NodePoolNames[0]) 37 | assert.Equal(t, "whoami", res.Commands[1].Command) 38 | 39 | assert.Len(t, res.VariableSchemas, 5) 40 | 41 | assert.NotNil(t, res.VariableSchemas["a"]) 42 | assert.False(t, res.VariableSchemas["a"].Sensitive) 43 | assert.False(t, res.VariableSchemas["a"].Optional) 44 | assert.False(t, res.VariableSchemas["a"].ReadOnly) 45 | assert.Equal(t, []string{"string"}, res.VariableSchemas["a"].Types) 46 | 47 | assert.NotNil(t, res.VariableSchemas["b"]) 48 | assert.False(t, res.VariableSchemas["b"].Sensitive) 49 | assert.False(t, res.VariableSchemas["b"].Optional) 50 | assert.True(t, res.VariableSchemas["b"].ReadOnly) 51 | assert.Equal(t, []string{"integer"}, res.VariableSchemas["b"].Types) 52 | 53 | assert.NotNil(t, res.VariableSchemas["c"]) 54 | assert.True(t, res.VariableSchemas["c"].Sensitive) 55 | assert.False(t, res.VariableSchemas["c"].Optional) 56 | assert.False(t, res.VariableSchemas["c"].ReadOnly) 57 | assert.Equal(t, []string{"string"}, res.VariableSchemas["c"].Types) 58 | 59 | assert.NotNil(t, res.VariableSchemas["d"]) 60 | assert.False(t, res.VariableSchemas["d"].Sensitive) 61 | assert.True(t, res.VariableSchemas["d"].Optional) 62 | assert.False(t, res.VariableSchemas["d"].ReadOnly) 63 | assert.Equal(t, []string{"boolean"}, res.VariableSchemas["d"].Types) 64 | } 65 | 66 | { // bad schema 67 | _, err := _package.LoadManifest(_fs, "tests/bad-schema.yaml") 68 | 69 | assert.Error(t, err) 70 | } 71 | } 72 | 73 | func TestValidateVarSet(t *testing.T) { 74 | manifest, _ := _package.LoadManifest(_fs, "tests/valid.yaml") 75 | 76 | { // valid 77 | vs := vars.VarSet{ 78 | "a": "aval", 79 | "c": "cval", 80 | "d": true, 81 | } 82 | 83 | res := manifest.ValidateVarSet(vs, true) 84 | 85 | assert.NoError(t, res) 86 | } 87 | 88 | { // read only 89 | vs := vars.VarSet{ 90 | "a": "aval", 91 | "b": "12", 92 | "c": "cval", 93 | "d": "true", 94 | } 95 | 96 | res := manifest.ValidateVarSet(vs, true) 97 | 98 | assert.Error(t, res) 99 | } 100 | 101 | { // read only no write 102 | vs := vars.VarSet{ 103 | "a": "aval", 104 | "b": 12, 105 | "c": "cval", 106 | "d": true, 107 | } 108 | 109 | res := manifest.ValidateVarSet(vs, false) 110 | 111 | assert.NoError(t, res) 112 | } 113 | 114 | { // optional 115 | vs := vars.VarSet{} 116 | 117 | res := manifest.ValidateVarSet(vs, true) 118 | 119 | assert.Error(t, res) 120 | } 121 | 122 | { // schema 123 | vs := vars.VarSet{ 124 | "a": "aval", 125 | "b": "five", 126 | "c": "cval", 127 | } 128 | 129 | res := manifest.ValidateVarSet(vs, true) 130 | 131 | assert.Error(t, res) 132 | } 133 | } 134 | 135 | func TestFilterVars(t *testing.T) { 136 | manifest, _ := _package.LoadManifest(_fs, "tests/valid.yaml") 137 | 138 | vs := vars.VarSet{ 139 | "a": "", 140 | "b": "", 141 | "c": "", 142 | "d": "", 143 | "e": "", 144 | "f": "", 145 | } 146 | 147 | res := manifest.FilterVars(vs) 148 | 149 | assert.Equal(t, res, vars.VarSet{"a": "", "b": "", "c": "", "d": "", "e": ""}) 150 | } 151 | 152 | func TestFilterSensitiveVars(t *testing.T) { 153 | manifest, _ := _package.LoadManifest(_fs, "tests/valid.yaml") 154 | 155 | vs := vars.VarSet{ 156 | "a": "", 157 | "b": "", 158 | "c": "", 159 | "d": "", 160 | "e": "", 161 | } 162 | 163 | res := manifest.FilterSensitiveVars(vs) 164 | 165 | assert.Equal(t, res, vars.VarSet{"a": "", "b": "", "d": "", "e": ""}) 166 | } 167 | -------------------------------------------------------------------------------- /pkg/package/package-manifest.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Corral Package Manifest", 3 | "type": "object", 4 | "required": ["name", "description"], 5 | "properties": { 6 | "name": { 7 | "type": "string", 8 | "minLength": 1, 9 | "maxLength": 255, 10 | "description": "The name of the package." 11 | }, 12 | "annotations": { 13 | "type": "object", 14 | "additionalProperties": {"type": "string"}, 15 | "description": "Additional information about this package." 16 | }, 17 | "description": { 18 | "type": "string", 19 | "description": "Describe the purpose of this package." 20 | }, 21 | "overlay": { 22 | "type": "object", 23 | "additionalProperties": {"type": "string"}, 24 | "description": "A map of node group name to overlay subpath." 25 | }, 26 | "commands": { 27 | "type": "array", 28 | "additionalProperties": false, 29 | "items": { 30 | "oneOf": [ 31 | { "$ref": "#/definitions/command" }, 32 | { "$ref": "#/definitions/module" } 33 | ] 34 | } 35 | }, 36 | "variables": { 37 | "type": "object", 38 | "additionalProperties": { 39 | "$ref": "#/definitions/variable" 40 | } 41 | } 42 | }, 43 | "definitions": { 44 | "command": { 45 | "type": "object", 46 | "required": ["command", "node_pools"], 47 | "properties": { 48 | "command": { 49 | "type": "string", 50 | "description": "The command to exec on every node in the listed pools." 51 | }, 52 | "node_pools": { 53 | "type": "array", 54 | "minLength": 1, 55 | "description": "A list of node pools to execute the command on.", 56 | "items": { 57 | "type": "string" 58 | } 59 | } 60 | } 61 | }, 62 | "module": { 63 | "type": "object", 64 | "required": ["module"], 65 | "properties": { 66 | "module": { 67 | "type": "string", 68 | "description": "Name of the terraform module." 69 | }, 70 | "skip_cleanup": { 71 | "type": "boolean", 72 | "default": false, 73 | "description": "Do not run terraform destroy when cleaning up a corral." 74 | } 75 | } 76 | }, 77 | "variable": { 78 | "type": "object", 79 | "$ref": "http://json-schema.org/draft-07/schema", 80 | "required": ["type"], 81 | "properties": { 82 | "sensitive": { 83 | "type": "boolean", 84 | "default": false, 85 | "description": "If a variable is marked as sensitive it will not be returned by the vars command or when a corral is exported." 86 | }, 87 | "optional": { 88 | "type": "boolean", 89 | "default": true, 90 | "description": "If a variable is marked as optional, it will not be required to use this package." 91 | }, 92 | "read_only": { 93 | "type": "boolean", 94 | "default": true, 95 | "description": "If a variable is marked as read only it can only returned by a corral, not set." 96 | }, 97 | "default": { 98 | "type": "object", 99 | "description": "If a variable has a default value and a value is not present for that variable, the default value will be used instead." 100 | } 101 | } 102 | } 103 | } 104 | } -------------------------------------------------------------------------------- /pkg/package/package.go: -------------------------------------------------------------------------------- 1 | package _package 2 | 3 | import ( 4 | "path/filepath" 5 | 6 | "github.com/rancherlabs/corral/pkg/version" 7 | ) 8 | 9 | var corralUserAgent = "Corral/" + version.Version 10 | 11 | const ( 12 | TerraformVersionAnnotation = "corral.cattle.io/terraform-version" 13 | PublisherAnnotation = "corral.cattle.io/published-by" 14 | CorralVersionAnnotation = "corral.cattle.io/corral-version" 15 | PublishTimestampAnnotation = "corral.cattle.io/published-at" 16 | ) 17 | 18 | type Package struct { 19 | RootPath string 20 | 21 | Manifest 22 | } 23 | 24 | func (b Package) TerraformVersion() string { 25 | v := b.Manifest.GetAnnotation(TerraformVersionAnnotation) 26 | 27 | if v == "" { 28 | v = version.TerraformVersion 29 | } 30 | 31 | return v 32 | } 33 | 34 | func (b Package) ManifestPath() string { 35 | return filepath.Join(b.RootPath, "manifest.yaml") 36 | } 37 | 38 | func (b Package) TerraformModulePath(name string) string { 39 | return filepath.Join(b.RootPath, "terraform", name) 40 | } 41 | 42 | func (b *Package) OverlayPath() string { 43 | return filepath.Join(b.RootPath, "overlay") 44 | } 45 | -------------------------------------------------------------------------------- /pkg/package/registry.go: -------------------------------------------------------------------------------- 1 | package _package 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/rancherlabs/corral/pkg/config" 7 | "oras.land/oras-go/pkg/auth" 8 | "oras.land/oras-go/pkg/auth/docker" 9 | dockerauth "oras.land/oras-go/pkg/auth/docker" 10 | "oras.land/oras-go/pkg/content" 11 | ) 12 | 13 | var registryCredentials = config.CorralRoot("registry-creds.json") 14 | 15 | func newRegistryStore() (reg content.Registry, err error) { 16 | authorizer, err := docker.NewClient(registryCredentials) 17 | if err != nil { 18 | return 19 | } 20 | 21 | headers := http.Header{} 22 | headers.Set("User-Agent", corralUserAgent) 23 | opts := []auth.ResolverOption{auth.WithResolverHeaders(headers)} 24 | resolver, err := authorizer.ResolverWithOpts(opts...) 25 | if err != nil { 26 | return 27 | } 28 | 29 | reg = content.Registry{Resolver: resolver} 30 | 31 | return 32 | } 33 | 34 | func AddRegistryCredentials(registry, username, password string) error { 35 | da, err := dockerauth.NewClient(registryCredentials) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | return da.LoginWithOpts(auth.WithLoginHostname(registry), 41 | auth.WithLoginUsername(username), 42 | auth.WithLoginSecret(password), 43 | auth.WithLoginUserAgent(corralUserAgent)) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/package/template.go: -------------------------------------------------------------------------------- 1 | package _package 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "io/fs" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/sirupsen/logrus" 13 | "gopkg.in/yaml.v3" 14 | ) 15 | 16 | type TemplateManifest struct { 17 | Name string `yaml:"name"` 18 | Annotations map[string]string `yaml:"annotations,omitempty"` 19 | Description string `yaml:"description"` 20 | Commands []Command `yaml:"commands"` 21 | Overlay map[string]string `yaml:"overlay,omitempty"` 22 | VariableSchemas map[string]any `yaml:"variables,omitempty"` 23 | } 24 | 25 | func Template(name, description string, packages ...string) error { 26 | pkgs := make([]Package, len(packages)) 27 | 28 | for i, p := range packages { 29 | pkg, err := LoadPackage(p) // ensures pkg is in cache 30 | if err != nil { 31 | return fmt.Errorf("failed to load [%s] package: %w", p, err) 32 | } 33 | pkgs[i] = pkg 34 | } 35 | 36 | manifest, err := MergePackages(name, description, pkgs) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | buf, err := yaml.Marshal(manifest) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | err = os.WriteFile(filepath.Join(name, "manifest.yaml"), buf, 0664) 47 | if err != nil { 48 | return fmt.Errorf("failed to write manifest: %w", err) 49 | } 50 | 51 | err = ValidateManifest(buf) 52 | if err != nil { 53 | return fmt.Errorf("rendered package is not a valid package: %w", err) 54 | } 55 | return nil 56 | } 57 | 58 | func MergePackages(name, description string, pkgs []Package) (TemplateManifest, error) { 59 | if description == "" { 60 | for i := range pkgs { 61 | if i > 0 { 62 | description += "\n" 63 | } 64 | 65 | if pkgs[i].Description != "" { 66 | description += pkgs[i].Description 67 | } 68 | } 69 | } 70 | 71 | t := TemplateManifest{ 72 | Name: filepath.Base(name), 73 | Description: description, 74 | Overlay: map[string]string{}, 75 | VariableSchemas: map[string]any{}, 76 | } 77 | 78 | for _, pkg := range pkgs { 79 | buf, err := os.ReadFile(filepath.Join(pkg.RootPath, "manifest.yaml")) 80 | if err != nil { 81 | return t, err 82 | } 83 | 84 | yml := struct { 85 | VariableSchemas map[string]any `yaml:"variables,omitempty"` 86 | }{ 87 | VariableSchemas: map[string]any{}, 88 | } 89 | 90 | err = yaml.Unmarshal(buf, &yml) 91 | if err != nil { 92 | return t, err 93 | } 94 | 95 | for _, c := range pkg.Commands { 96 | if c.Module != "" { 97 | c.Module = filepath.Join(pkg.Name, c.Module) 98 | } 99 | t.Commands = append(t.Commands, c) 100 | } 101 | for k, v := range pkg.Overlay { 102 | t.Overlay[k] = v 103 | } 104 | for k, v := range yml.VariableSchemas { 105 | if _, ok := yml.VariableSchemas[k]; ok { 106 | t.VariableSchemas[k] = mergeVariable(yml.VariableSchemas[k], v) 107 | } else { 108 | t.VariableSchemas[k] = v 109 | } 110 | } 111 | 112 | logrus.Infof("Copying modules from %s", pkg.Name) 113 | 114 | err = copyTerraform(name, pkg) 115 | if err != nil { 116 | return t, err 117 | } 118 | 119 | logrus.Infof("Copying overlay from %s", pkg.Name) 120 | 121 | err = copyOverlay(name, pkg) 122 | if err != nil { 123 | return t, err 124 | } 125 | } 126 | return t, nil 127 | } 128 | 129 | func copyTerraform(root string, pkg Package) error { 130 | return copyFiles(filepath.Join(root, "terraform", pkg.Name), pkg.TerraformModulePath(""), pkg) 131 | } 132 | 133 | func copyOverlay(root string, pkg Package) error { 134 | return copyFiles(filepath.Join(root, "overlay"), pkg.OverlayPath(), pkg) 135 | } 136 | 137 | func copyFiles(root, dir string, pkg Package) error { 138 | _, err := os.Stat(dir) 139 | if errors.Is(err, os.ErrNotExist) { 140 | return nil 141 | } 142 | err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { 143 | if err != nil { 144 | return err 145 | } 146 | 147 | orig := path[len(dir):] 148 | destPath := root + orig 149 | 150 | if d.IsDir() { 151 | logrus.Debugf("creating %s", destPath) 152 | if err = os.MkdirAll(destPath, 0700); err != nil { 153 | return err 154 | } 155 | } else { 156 | // skip manifests 157 | if strings.HasSuffix(path, "manifest.yaml") { 158 | return nil 159 | } 160 | logrus.Debugf("%s: %s -> %s", pkg.Name, orig, destPath) 161 | 162 | if _, err = os.Stat(destPath); err == nil { 163 | _ = os.Remove(destPath) 164 | } 165 | 166 | f, err := os.Create(destPath) 167 | if err != nil { 168 | return err 169 | } 170 | 171 | in, err := os.Open(path) 172 | if err != nil { 173 | return err 174 | } 175 | 176 | _, err = io.Copy(f, in) 177 | _ = f.Close() 178 | _ = in.Close() 179 | 180 | if err != nil { 181 | return err 182 | } 183 | } 184 | 185 | return nil 186 | }) 187 | return err 188 | } 189 | 190 | func mergeVariable(vars ...any) any { 191 | out := map[string]any{} 192 | 193 | for _, v := range vars { 194 | vm := v.(map[string]any) 195 | 196 | for k, v := range vm { 197 | out[k] = v 198 | } 199 | } 200 | 201 | return out 202 | 203 | } 204 | -------------------------------------------------------------------------------- /pkg/package/tests/bad-schema.yaml: -------------------------------------------------------------------------------- 1 | name: bad-schema 2 | -------------------------------------------------------------------------------- /pkg/package/tests/valid.yaml: -------------------------------------------------------------------------------- 1 | name: valid 2 | description: "valid description" 3 | annotations: 4 | foo: bar 5 | baz: "1" 6 | overlay: 7 | foo: bastion 8 | commands: 9 | - module: module 10 | skip_cleanup: true 11 | - node_pools: 12 | - foo 13 | - bar 14 | command: whoami 15 | variables: 16 | a: 17 | type: string 18 | b: 19 | type: integer 20 | readOnly: true 21 | c: 22 | sensitive: true 23 | type: string 24 | d: 25 | type: boolean 26 | optional: true 27 | e: 28 | type: string 29 | default: test 30 | -------------------------------------------------------------------------------- /pkg/package/upload.go: -------------------------------------------------------------------------------- 1 | package _package 2 | 3 | import ( 4 | "archive/tar" 5 | "bytes" 6 | "compress/gzip" 7 | "context" 8 | "io" 9 | "io/fs" 10 | "os" 11 | "path/filepath" 12 | 13 | v1 "github.com/opencontainers/image-spec/specs-go/v1" 14 | "github.com/sirupsen/logrus" 15 | "oras.land/oras-go/pkg/content" 16 | "oras.land/oras-go/pkg/oras" 17 | ) 18 | 19 | func UploadPackage(pkg Package, ref string) error { 20 | logrus.Info("building manifest") 21 | memoryStore := content.NewMemory() 22 | configDescriptor, err := getManifestDescriptor(memoryStore, pkg) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | logrus.Info("compressing package contents") 28 | var contents []v1.Descriptor 29 | 30 | if desc, err := addManifestLayer(memoryStore, pkg); err == nil { 31 | contents = append(contents, desc) 32 | } else { 33 | return err 34 | } 35 | 36 | if ds, err := addTerraformModuleLayers(memoryStore, pkg); err == nil { 37 | contents = append(contents, ds...) 38 | } else { 39 | return err 40 | } 41 | 42 | if desc, err := addOverlayLayer(memoryStore, pkg); err == nil { 43 | contents = append(contents, desc) 44 | } else { 45 | return err 46 | } 47 | 48 | manifestData, manifestDescriptor, err := content.GenerateManifest(&configDescriptor, pkg.Annotations, contents...) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | if err = memoryStore.StoreManifest(ref, manifestDescriptor, manifestData); err != nil { 54 | return err 55 | } 56 | 57 | registryStore, err := newRegistryStore() 58 | if err != nil { 59 | return err 60 | } 61 | 62 | logrus.Info("pushing to registry") 63 | _, err = oras.Copy(context.Background(), memoryStore, ref, registryStore, "") 64 | return err 65 | } 66 | 67 | func getManifestDescriptor(memoryStore *content.Memory, pkg Package) (v1.Descriptor, error) { 68 | buf, err := os.ReadFile(pkg.ManifestPath()) 69 | if err != nil { 70 | return v1.Descriptor{}, err 71 | } 72 | 73 | return memoryStore.Add("", v1.MediaTypeImageLayer, buf) 74 | } 75 | 76 | func addManifestLayer(memoryStore *content.Memory, pkg Package) (v1.Descriptor, error) { 77 | manifest, err := os.ReadFile(pkg.ManifestPath()) 78 | if err != nil { 79 | return v1.Descriptor{}, err 80 | } 81 | 82 | var buf bytes.Buffer 83 | gz := gzip.NewWriter(&buf) 84 | tw := tar.NewWriter(gz) 85 | 86 | hdr := &tar.Header{ 87 | Name: "manifest.yaml", 88 | Size: int64(len(manifest)), 89 | } 90 | 91 | if err = tw.WriteHeader(hdr); err != nil { 92 | return v1.Descriptor{}, err 93 | } 94 | 95 | if _, err = tw.Write(manifest); err != nil { 96 | return v1.Descriptor{}, err 97 | } 98 | 99 | err = tw.Close() 100 | if err != nil { 101 | return v1.Descriptor{}, err 102 | } 103 | 104 | err = gz.Close() 105 | if err != nil { 106 | return v1.Descriptor{}, err 107 | } 108 | 109 | return memoryStore.Add("", v1.MediaTypeImageLayerGzip, buf.Bytes()) 110 | } 111 | 112 | func addTerraformModuleLayers(memoryStore *content.Memory, pkg Package) ([]v1.Descriptor, error) { 113 | var ds []v1.Descriptor 114 | 115 | var desc v1.Descriptor 116 | for _, cmd := range pkg.Commands { 117 | if cmd.Module != "" { 118 | buf, err := compressPath(filepath.Join("terraform", cmd.Module), pkg.TerraformModulePath(cmd.Module)) 119 | if err != nil { 120 | return nil, err 121 | } 122 | 123 | desc, err = memoryStore.Add("", v1.MediaTypeImageLayerGzip, buf) 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | ds = append(ds, desc) 129 | } 130 | } 131 | 132 | return ds, nil 133 | } 134 | 135 | func addOverlayLayer(memoryStore *content.Memory, pkg Package) (v1.Descriptor, error) { 136 | buf, err := compressPath("overlay", pkg.OverlayPath()) 137 | if err != nil { 138 | return v1.Descriptor{}, err 139 | } 140 | 141 | return memoryStore.Add("", v1.MediaTypeImageLayerGzip, buf) 142 | } 143 | 144 | func compressPath(prefix, root string) ([]byte, error) { 145 | var buf bytes.Buffer 146 | gz := gzip.NewWriter(&buf) 147 | tw := tar.NewWriter(gz) 148 | 149 | sfs := os.DirFS(root) 150 | err := fs.WalkDir(sfs, ".", func(path string, d fs.DirEntry, err error) error { 151 | if path == "." { 152 | return nil 153 | } 154 | 155 | if err != nil { 156 | return err 157 | } 158 | 159 | if d.IsDir() { 160 | hdr := &tar.Header{ 161 | Name: filepath.Join(prefix, path), 162 | Typeflag: tar.TypeDir, 163 | } 164 | if err = tw.WriteHeader(hdr); err != nil { 165 | return err 166 | } 167 | return nil 168 | } 169 | 170 | stat, err := os.Stat(filepath.Join(root, path)) 171 | if err != nil { 172 | return err 173 | } 174 | 175 | f, err := os.Open(filepath.Join(root, path)) 176 | if err != nil { 177 | return err 178 | } 179 | defer func(f *os.File) { _ = f.Close() }(f) 180 | 181 | hdr := &tar.Header{ 182 | Name: filepath.Join(prefix, path), 183 | Size: stat.Size(), 184 | } 185 | 186 | if err = tw.WriteHeader(hdr); err != nil { 187 | return err 188 | } 189 | 190 | _, err = io.Copy(tw, f) 191 | if err != nil { 192 | return err 193 | } 194 | 195 | return nil 196 | }) 197 | if err != nil { 198 | return nil, err 199 | } 200 | 201 | err = tw.Close() 202 | if err != nil { 203 | return nil, err 204 | } 205 | 206 | err = gz.Close() 207 | if err != nil { 208 | return nil, err 209 | } 210 | 211 | return buf.Bytes(), nil 212 | } 213 | -------------------------------------------------------------------------------- /pkg/package/validate.go: -------------------------------------------------------------------------------- 1 | package _package 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | var ErrOverlayNotFound = errors.New("overlay folder not found") 11 | 12 | func Validate(name string) error { 13 | pkg, err := LoadPackage(name) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | if i, err := os.Stat(pkg.OverlayPath()); err != nil || !i.IsDir() { 19 | return ErrOverlayNotFound 20 | } 21 | 22 | for _, cmd := range pkg.Commands { 23 | if cmd.Module != "" { 24 | if i, err := os.Stat(pkg.TerraformModulePath(cmd.Module)); err != nil || !i.IsDir() { 25 | return err 26 | } 27 | } 28 | } 29 | 30 | err = pkg.ValidateDefaults() 31 | if err != nil { 32 | return err 33 | } 34 | 35 | logrus.Info("package is valid") 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /pkg/package/variable.go: -------------------------------------------------------------------------------- 1 | package _package 2 | 3 | import ( 4 | "github.com/santhosh-tekuri/jsonschema/v5" 5 | ) 6 | 7 | type Schema struct { 8 | *jsonschema.Schema 9 | 10 | Sensitive bool 11 | Optional bool 12 | Description string 13 | Default any 14 | } 15 | -------------------------------------------------------------------------------- /pkg/shell/registry.go: -------------------------------------------------------------------------------- 1 | package shell 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/rancherlabs/corral/pkg/corral" 8 | "github.com/rancherlabs/corral/pkg/vars" 9 | "k8s.io/apimachinery/pkg/util/wait" 10 | ) 11 | 12 | type Registry struct { 13 | reg *sync.Map 14 | } 15 | 16 | func NewRegistry() *Registry { 17 | return &Registry{ 18 | reg: &sync.Map{}, 19 | } 20 | } 21 | 22 | // GetShell will return the shell associated with the given node's address. If the shell does not exist one will be 23 | // created. 24 | func (r *Registry) GetShell(n corral.Node, privateKey string, vs vars.VarSet) (*Shell, error) { 25 | var err error 26 | 27 | if sh, ok := r.reg.Load(n.Address); ok { 28 | return sh.(*Shell), nil 29 | } 30 | 31 | err = wait.Poll(time.Second, 2*time.Minute, func() (done bool, err error) { 32 | sh := &Shell{ 33 | Node: n, 34 | PrivateKey: []byte(privateKey), 35 | Vars: vs, 36 | } 37 | 38 | if err = sh.Connect(); err != nil { 39 | sh.Close() 40 | return false, nil 41 | } 42 | 43 | r.reg.Store(n.Address, sh) 44 | 45 | return err == nil, err 46 | }) 47 | 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | sh, _ := r.reg.Load(n.Address) 53 | return sh.(*Shell), err 54 | } 55 | 56 | // Close all shells in the registry. 57 | func (r *Registry) Close() { 58 | r.reg.Range(func(key, value any) bool { 59 | value.(*Shell).Close() 60 | 61 | return true 62 | }) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/shell/shell.go: -------------------------------------------------------------------------------- 1 | package shell 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/fs" 9 | "net" 10 | "os" 11 | "path/filepath" 12 | "sort" 13 | "strings" 14 | "sync" 15 | "time" 16 | 17 | "github.com/rancherlabs/corral/pkg/vars" 18 | 19 | "github.com/pkg/sftp" 20 | "github.com/rancherlabs/corral/pkg/corral" 21 | _package "github.com/rancherlabs/corral/pkg/package" 22 | "github.com/sirupsen/logrus" 23 | "golang.org/x/crypto/ssh" 24 | ) 25 | 26 | const ( 27 | connectionTimeout = 5 * time.Second 28 | 29 | corralSetVarCommand = "corral_set" 30 | corralLogMessageCommand = "corral_log" 31 | ) 32 | 33 | type Shell struct { 34 | Node corral.Node 35 | PrivateKey []byte 36 | Vars vars.VarSet 37 | 38 | sftpClient *sftp.Client 39 | bastionClient *ssh.Client 40 | client *ssh.Client 41 | connection net.Conn 42 | } 43 | 44 | func (s *Shell) Connect() error { 45 | if len(strings.Split(s.Node.Address, ":")) == 1 { 46 | s.Node.Address += ":22" 47 | } 48 | 49 | if s.Node.BastionAddress != "" && len(strings.Split(s.Node.BastionAddress, ":")) == 1 { 50 | s.Node.BastionAddress += ":22" 51 | } 52 | 53 | signer, err := ssh.ParsePrivateKey(s.PrivateKey) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | sshConfig := ssh.ClientConfig{ 59 | User: s.Node.User, 60 | Timeout: connectionTimeout, 61 | Auth: []ssh.AuthMethod{ 62 | ssh.PublicKeys(signer), 63 | }, 64 | HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { 65 | return nil 66 | }, 67 | } 68 | 69 | // establish a connection to the server 70 | if s.Node.BastionAddress != "" { 71 | s.bastionClient, err = ssh.Dial("tcp", s.Node.BastionAddress, &sshConfig) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | s.connection, err = s.bastionClient.Dial("tcp", s.Node.Address) 77 | if err != nil { 78 | return err 79 | } 80 | } else { 81 | s.connection, err = net.DialTimeout("tcp", s.Node.Address, connectionTimeout) 82 | if err != nil { 83 | return err 84 | } 85 | } 86 | 87 | // upgrade connection to ssh connection 88 | sshConn, cc, cr, err := ssh.NewClientConn(s.connection, s.Node.Address, &sshConfig) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | // create ssh client 94 | s.client = ssh.NewClient(sshConn, cc, cr) 95 | 96 | // connect sftp client 97 | s.sftpClient, err = sftp.NewClient(s.client) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | // test sftp connection 103 | _, err = s.sftpClient.Stat("/") 104 | if err != nil { 105 | return err 106 | } 107 | 108 | return nil 109 | } 110 | 111 | func (s *Shell) UploadPackageFiles(pkg _package.Package) error { 112 | src := pkg.OverlayPath() 113 | if len(s.Node.OverlayRoot) > 0 { 114 | src = filepath.Join(src, s.Node.OverlayRoot) 115 | } 116 | 117 | return filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { 118 | if err != nil { 119 | return err 120 | } 121 | 122 | dest := path[len(src):] 123 | 124 | if dest == "" { 125 | return nil 126 | } 127 | 128 | if info.IsDir() { 129 | return s.sftpClient.MkdirAll(dest) 130 | } 131 | 132 | in, err := os.Open(path) 133 | if err != nil { 134 | return err 135 | } 136 | defer func() { _ = in.Close() }() 137 | 138 | out, err := s.sftpClient.Create(dest) 139 | if err != nil { 140 | return err 141 | } 142 | defer func() { _ = out.Close() }() 143 | 144 | err = out.Chmod(0o700) 145 | if err != nil { 146 | return err 147 | } 148 | 149 | logrus.Debugf("copying %s to [%s]:%s", path, s.Node.Name, dest) 150 | 151 | _, err = io.Copy(out, in) 152 | if err != nil { 153 | return err 154 | } 155 | 156 | return nil 157 | }) 158 | } 159 | 160 | func (s *Shell) Run(c string) error { 161 | session, err := s.client.NewSession() 162 | if err != nil { 163 | return err 164 | } 165 | 166 | stdout, _ := session.StdoutPipe() 167 | stderr, _ := session.StderrPipe() 168 | 169 | var wg sync.WaitGroup 170 | wg.Add(2) 171 | go func() { 172 | s.consumeStdout(stdout) 173 | wg.Done() 174 | }() 175 | go func() { 176 | s.consumeStderr(stderr) 177 | wg.Done() 178 | }() 179 | 180 | envVars, err := varsToEnvVars(s.Vars) 181 | if err != nil { 182 | return err 183 | } 184 | request := strings.Join(append(envVars, c), "\n") 185 | 186 | logrus.Tracef("request: %s", request) 187 | 188 | err = session.Run(request) 189 | wg.Wait() 190 | 191 | return err 192 | } 193 | 194 | func varsToEnvVars(varSet vars.VarSet) ([]string, error) { 195 | result := make([]string, 0, len(varSet)) 196 | keys := make([]string, 0, len(varSet)) 197 | for k := range varSet { 198 | keys = append(keys, k) 199 | } 200 | sort.Strings(keys) 201 | 202 | for _, k := range keys { 203 | v := varSet[k] 204 | b, err := json.Marshal(&v) 205 | if err != nil { 206 | return nil, err 207 | } 208 | str := string(b) 209 | if strings.HasPrefix(str, `"`) && strings.HasSuffix(str, `"`) { 210 | str = strings.Trim(str, `"`) 211 | } 212 | if !strings.HasPrefix(str, `'`) && !strings.HasSuffix(str, `'`) { 213 | str = fmt.Sprintf("'%s'", str) 214 | } 215 | result = append(result, fmt.Sprintf("export CORRAL_%s=%s", k, str)) 216 | result = append(result, fmt.Sprintf("export TF_VAR_%s=%s", k, str)) 217 | } 218 | return result, nil 219 | } 220 | 221 | func (s *Shell) Close() { 222 | if s.sftpClient != nil { 223 | _ = s.sftpClient.Close() 224 | } 225 | 226 | if s.connection != nil { 227 | _ = s.connection.Close() 228 | } 229 | 230 | if s.bastionClient != nil { 231 | _ = s.bastionClient.Close() 232 | } 233 | } 234 | 235 | func (s *Shell) consumeStdout(pipe io.Reader) { 236 | scanner := bufio.NewScanner(pipe) 237 | 238 | for scanner.Scan() { 239 | text := scanner.Text() 240 | 241 | if strings.HasPrefix(text, corralSetVarCommand) { 242 | cmd := strings.TrimPrefix(text, corralSetVarCommand) 243 | cmd = strings.Trim(cmd, " \t") 244 | 245 | k, v, err := vars.ToVar(cmd) 246 | if err != nil { 247 | logrus.Error(err) 248 | } 249 | if k == "" { 250 | logrus.Warnf("failed to parse corral command: %s", text) 251 | continue 252 | } 253 | 254 | s.Vars[k] = v 255 | } else if strings.HasPrefix(text, corralLogMessageCommand) { 256 | vs := strings.TrimPrefix(text, corralLogMessageCommand) 257 | vs = strings.Trim(vs, " \t") 258 | 259 | logrus.Info(vs) 260 | } 261 | 262 | logrus.Debugf("[%s]: %s", s.Node.Name, text) 263 | } 264 | } 265 | 266 | func (s *Shell) consumeStderr(pipe io.Reader) { 267 | scanner := bufio.NewScanner(pipe) 268 | 269 | for scanner.Scan() { 270 | logrus.Debugf("[%s]: %s", s.Node.Name, scanner.Text()) 271 | } 272 | } 273 | -------------------------------------------------------------------------------- /pkg/shell/shell_test.go: -------------------------------------------------------------------------------- 1 | package shell 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/rancherlabs/corral/pkg/vars" 9 | "github.com/stretchr/testify/require" 10 | "gotest.tools/v3/assert" 11 | ) 12 | 13 | func TestVarsToEnvVars(t *testing.T) { 14 | tests := []struct { 15 | name string 16 | input vars.VarSet 17 | expected []string 18 | }{ 19 | { 20 | name: "int", 21 | input: map[string]any{ 22 | "int": 1, 23 | }, 24 | expected: []string{ 25 | "export CORRAL_int='1'", 26 | }, 27 | }, 28 | { 29 | name: "string", 30 | input: map[string]any{ 31 | "string": "test", 32 | }, 33 | expected: []string{ 34 | "export CORRAL_string='test'", 35 | }, 36 | }, 37 | { 38 | name: "empty array", 39 | input: map[string]any{ 40 | "array": []string{}, 41 | }, 42 | expected: []string{ 43 | `export CORRAL_array='[]'`, 44 | }, 45 | }, 46 | { 47 | name: "array of strings", 48 | input: map[string]any{ 49 | "array": []string{"a", "b", "c"}, 50 | }, 51 | expected: []string{ 52 | `export CORRAL_array='["a","b","c"]'`, 53 | }, 54 | }, 55 | { 56 | name: "array of numbers", 57 | input: map[string]any{ 58 | "array": []int{1, 2, 3}, 59 | }, 60 | expected: []string{ 61 | `export CORRAL_array='[1,2,3]'`, 62 | }, 63 | }, 64 | { 65 | name: "empty object", 66 | input: map[string]any{ 67 | "object": map[string]any{}, 68 | }, 69 | expected: []string{ 70 | `export CORRAL_object='{}'`, 71 | }, 72 | }, 73 | { 74 | name: "object", 75 | input: map[string]any{ 76 | "object": map[string]any{ 77 | "1": "a", 78 | "2": 2, 79 | "3": []any{4.1, "5"}, 80 | }, 81 | }, 82 | expected: []string{ 83 | `export CORRAL_object='{"1":"a","2":2,"3":[4.1,"5"]}'`, 84 | }, 85 | }, 86 | } 87 | 88 | for _, tt := range tests { 89 | t.Run(tt.name, func(t *testing.T) { 90 | actual, err := varsToEnvVars(tt.input) 91 | require.NoError(t, err) 92 | 93 | assert.DeepEqual(t, actual, tt.expected) 94 | }) 95 | } 96 | } 97 | 98 | func TestConsumeStdout(t *testing.T) { 99 | tests := []struct { 100 | name string 101 | input string 102 | expected any 103 | }{ 104 | { 105 | name: "unqouted string", 106 | input: "a", 107 | expected: "a", 108 | }, 109 | { 110 | name: "single qouted string", 111 | input: "'a'", 112 | expected: "'a'", 113 | }, 114 | { 115 | name: "double qouted string", 116 | input: `"a"`, 117 | expected: "a", 118 | }, 119 | { 120 | name: "unqouted number", 121 | input: "1", 122 | expected: 1., 123 | }, 124 | { 125 | name: "single qouted number", 126 | input: "'1'", 127 | expected: "'1'", 128 | }, 129 | { 130 | name: "double qouted number", 131 | input: `"1"`, 132 | expected: "1", 133 | }, 134 | { 135 | name: "homogeneously typed array", 136 | input: `[1,2,3]`, 137 | expected: []any{1., 2., 3.}, 138 | }, 139 | { 140 | name: "variously typed array", 141 | input: `["1",2,3.1]`, 142 | expected: []any{"1", 2., 3.1}, 143 | }, 144 | { 145 | name: "object", 146 | input: `{"a":1,"b":"2","c":["1",2.0]}`, 147 | expected: map[string]any{"a": 1., "b": "2", "c": []any{"1", 2.}}, 148 | }, 149 | } 150 | 151 | for _, tt := range tests { 152 | t.Run(tt.name, func(t *testing.T) { 153 | var b bytes.Buffer 154 | s := Shell{ 155 | Vars: map[string]any{}, 156 | } 157 | b.WriteString(fmt.Sprintf("corral_set test=%s\n", tt.input)) 158 | s.consumeStdout(&b) 159 | assert.DeepEqual(t, s.Vars["test"], tt.expected) 160 | }) 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /pkg/vars/vars.go: -------------------------------------------------------------------------------- 1 | package vars 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "regexp" 8 | "strings" 9 | 10 | "github.com/hashicorp/terraform-exec/tfexec" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | const quote = byte('"') 15 | 16 | // if it doesn't start with a { or [, or if it is entirely numbers 17 | var regex = regexp.MustCompile(`(^([\[{])+)|(^\d+$)`) 18 | 19 | // ToVar parses a var string and returns the key and value. 20 | func ToVar(in string) (key string, value any, err error) { 21 | parts := strings.SplitN(in, "=", 2) 22 | if len(parts) != 2 || len(parts[1]) == 0 { 23 | return parts[0], nil, nil 24 | } 25 | 26 | key = parts[0] 27 | 28 | value, err = FromJson(parts[1]) 29 | if err != nil { 30 | return "", nil, err 31 | } 32 | 33 | return 34 | } 35 | 36 | func FromJson(in string) (value any, err error) { 37 | // raw string values need to be quoted, so any value that doesn't start with a { or [, or is entirely numbers is 38 | // assumed a string 39 | // todo(jhyde): allow specifying types when using corral_set if necessary (i.e. variable is known, or e.g. "1") 40 | if !regex.Match([]byte(in)) && !strings.HasPrefix(in, `"`) && !strings.HasSuffix(in, `"`) { 41 | in = fmt.Sprintf(`"%s"`, in) 42 | } 43 | 44 | err = json.Unmarshal([]byte(in), &value) 45 | if err != nil { 46 | return nil, errors.Wrapf(err, `unmarshaling "%s"`, in) 47 | } 48 | 49 | return 50 | } 51 | 52 | // FromTerraformOutputMeta returns strings as they are and properly escapes json objects 53 | func FromTerraformOutputMeta(in tfexec.OutputMeta) (any, error) { 54 | raw, _ := in.Value.MarshalJSON() 55 | var m any 56 | 57 | err := json.Unmarshal(raw, &m) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | return m, err 63 | } 64 | 65 | func Escape(buf *bytes.Buffer) (out string) { 66 | c, err := buf.ReadByte() 67 | if err != nil { 68 | return 69 | } 70 | 71 | for { 72 | if c == quote { 73 | out += `\` 74 | } 75 | 76 | out += string(c) 77 | 78 | c, err = buf.ReadByte() 79 | if err != nil { 80 | return 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /pkg/vars/vars_test.go: -------------------------------------------------------------------------------- 1 | package vars 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/hashicorp/terraform-exec/tfexec" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestToVar(t *testing.T) { 12 | type args struct { 13 | in string 14 | } 15 | tests := []struct { 16 | name string 17 | args args 18 | wantKey string 19 | wantValue any 20 | }{ 21 | { 22 | name: "valid", 23 | args: struct{ in string }{in: "foo=bar"}, 24 | wantKey: "foo", 25 | wantValue: "bar", 26 | }, 27 | { 28 | name: "valid ip", 29 | args: struct{ in string }{in: `foo=127.0.0.1`}, 30 | wantKey: "foo", 31 | wantValue: "127.0.0.1", 32 | }, 33 | { 34 | name: "valid int", 35 | args: struct{ in string }{in: "foo=1"}, 36 | wantKey: "foo", 37 | wantValue: 1., 38 | }, 39 | { 40 | name: "json map", 41 | args: struct{ in string }{in: `foo={"test":1}`}, 42 | wantKey: "foo", 43 | wantValue: map[string]any{"test": 1.}, 44 | }, 45 | { 46 | name: "json slice", 47 | args: struct{ in string }{in: `foo=[1, 2, 3]`}, 48 | wantKey: "foo", 49 | wantValue: []any{1., 2., 3.}, 50 | }, 51 | { 52 | name: "json object", 53 | args: struct{ in string }{in: `foo={"test":{"a":1,"b":"2"}}`}, 54 | wantKey: "foo", 55 | wantValue: map[string]any{"test": map[string]any{"a": 1., "b": "2"}}, 56 | }, 57 | { 58 | name: "empty value", 59 | args: struct{ in string }{in: "foo="}, 60 | wantKey: "foo", 61 | wantValue: nil, 62 | }, 63 | { 64 | name: "missing value", 65 | args: struct{ in string }{in: "foo"}, 66 | wantKey: "foo", 67 | wantValue: nil, 68 | }, 69 | { 70 | name: "empty", 71 | args: struct{ in string }{in: ""}, 72 | wantKey: "", 73 | wantValue: nil, 74 | }, 75 | } 76 | 77 | for _, tt := range tests { 78 | t.Run(tt.name, func(t *testing.T) { 79 | gotKey, gotValue, err := ToVar(tt.args.in) 80 | assert.NoError(t, err) 81 | if gotKey != tt.wantKey { 82 | t.Errorf("ToVar() gotKey = %v, want %v", gotKey, tt.wantKey) 83 | } 84 | if !Equal(gotValue, tt.wantValue) { 85 | t.Errorf("ToVar() gotValue = %v, want %v", gotValue, tt.wantValue) 86 | } 87 | }) 88 | } 89 | } 90 | 91 | func TestFromTerraformOutputMeta(t *testing.T) { 92 | type args struct { 93 | in tfexec.OutputMeta 94 | } 95 | tests := []struct { 96 | name string 97 | args args 98 | want any 99 | }{ 100 | { 101 | name: "int", 102 | args: struct{ in tfexec.OutputMeta }{in: tfexec.OutputMeta{ 103 | Value: []byte(`1`), 104 | }}, 105 | want: 1., 106 | }, 107 | { 108 | name: "string", 109 | args: struct{ in tfexec.OutputMeta }{in: tfexec.OutputMeta{ 110 | Value: []byte(`"foo"`), 111 | }}, 112 | want: `foo`, 113 | }, 114 | { 115 | name: "array", 116 | args: struct{ in tfexec.OutputMeta }{in: tfexec.OutputMeta{ 117 | Value: []byte(`["a","b","c"]`), 118 | }}, 119 | want: []string{"a", "b", "c"}, 120 | }, 121 | { 122 | name: "object", 123 | args: struct{ in tfexec.OutputMeta }{in: tfexec.OutputMeta{ 124 | Value: []byte(`{"a":"b"}`), 125 | }}, 126 | want: map[string]any{"a": "b"}, 127 | }, 128 | { 129 | name: "nested object", 130 | args: struct{ in tfexec.OutputMeta }{in: tfexec.OutputMeta{ 131 | Value: []byte(`{"a":{"b":2}}`), 132 | }}, 133 | want: map[string]any{ 134 | "a": map[string]any{ 135 | "b": 2., 136 | }, 137 | }, 138 | }, 139 | } 140 | for _, tt := range tests { 141 | t.Run(tt.name, func(t *testing.T) { 142 | if got, err := FromTerraformOutputMeta(tt.args.in); err != nil { 143 | t.Error(err) 144 | } else { 145 | if !Equal(got, tt.want) { 146 | t.Errorf("FromTerraformOutputMeta() = %v, want %v", got, tt.want) 147 | } 148 | } 149 | }) 150 | } 151 | } 152 | 153 | func Equal(got, wanted any) bool { 154 | if reflect.TypeOf(got) == reflect.TypeOf(wanted) { 155 | return reflect.DeepEqual(got, wanted) 156 | } 157 | 158 | if got == nil || wanted == nil { 159 | return got == wanted 160 | } 161 | 162 | kind := reflect.TypeOf(got).Kind() 163 | 164 | if kind != reflect.TypeOf(wanted).Kind() { 165 | return false 166 | } 167 | 168 | gotValue := reflect.ValueOf(got) 169 | wantedValue := reflect.ValueOf(wanted) 170 | 171 | switch kind { 172 | case reflect.Slice: 173 | if gotValue.Len() != wantedValue.Len() { 174 | return false 175 | } 176 | for i := 0; i < gotValue.Len(); i++ { 177 | if !Equal(gotValue.Index(i).Interface(), wantedValue.Index(i).Interface()) { 178 | return false 179 | } 180 | } 181 | return true 182 | case reflect.Map: 183 | if gotValue.Len() != wantedValue.Len() { 184 | return false 185 | } 186 | for _, k := range gotValue.MapKeys() { 187 | val1 := gotValue.MapIndex(k) 188 | val2 := wantedValue.MapIndex(k) 189 | if !val1.IsValid() || !val2.IsValid() || !Equal(val1.Interface(), val2.Interface()) { 190 | return false 191 | } 192 | } 193 | return true 194 | } 195 | 196 | return false 197 | } 198 | -------------------------------------------------------------------------------- /pkg/vars/varset.go: -------------------------------------------------------------------------------- 1 | package vars 2 | 3 | type VarSet map[string]any 4 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | var Version = "0.0.0-dev" 4 | 5 | const TerraformVersion = "1.0.11" 6 | -------------------------------------------------------------------------------- /tests/integration/complex-types/corral_simple_output_test.go: -------------------------------------------------------------------------------- 1 | package complex_types 2 | 3 | import ( 4 | "crypto/rand" 5 | "crypto/rsa" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | 10 | "github.com/rancherlabs/corral/cmd" 11 | cmdconfig "github.com/rancherlabs/corral/cmd/config" 12 | cmdpackage "github.com/rancherlabs/corral/cmd/package" 13 | "github.com/rancherlabs/corral/pkg/config" 14 | "github.com/rancherlabs/corral/pkg/corral" 15 | "github.com/rancherlabs/corral/pkg/vars" 16 | "github.com/stretchr/testify/require" 17 | "golang.org/x/crypto/ssh" 18 | "gotest.tools/v3/assert" 19 | ) 20 | 21 | func TestSimpleOutput(t *testing.T) { 22 | config.InitializeRootPath(t.TempDir()) 23 | t.Run("validate", func(t *testing.T) { 24 | validateCmd := cmdpackage.NewCommandValidate() 25 | validateCmd.SetArgs([]string{"testdata"}) 26 | require.NoError(t, validateCmd.Execute()) 27 | }) 28 | configCmd := cmdconfig.NewCommandConfig() 29 | priv, err := rsa.GenerateKey(rand.Reader, 4096) 30 | require.NoError(t, err) 31 | require.NoError(t, priv.Validate()) 32 | 33 | pub, err := ssh.NewPublicKey(&priv.PublicKey) 34 | require.NoError(t, err) 35 | 36 | pubBytes := ssh.MarshalAuthorizedKey(pub) 37 | pubPath := filepath.Join(t.TempDir(), "id_rsa.pub") 38 | require.NoError(t, os.WriteFile(pubPath, pubBytes, 0o600)) 39 | 40 | configCmd.SetArgs([]string{"--user_id", "testuser", "--public_key", pubPath}) 41 | require.NoError(t, configCmd.Execute()) 42 | t.Cleanup(func() { 43 | deleteCmd := cmd.NewCommandDelete() 44 | deleteCmd.SetArgs([]string{"test-corral"}) 45 | require.NoError(t, deleteCmd.Execute()) 46 | }) 47 | t.Run("create", func(t *testing.T) { 48 | createCmd := cmd.NewCommandCreate() 49 | createCmd.SetArgs([]string{"test-corral", "testdata"}) 50 | require.NoError(t, createCmd.Execute()) 51 | t.Run("variables", func(t *testing.T) { 52 | c, err := corral.Load(config.CorralPath("test-corral")) 53 | require.NoError(t, err) 54 | 55 | tests := []struct { 56 | name string 57 | expected any 58 | }{ 59 | { 60 | name: "number", 61 | expected: 1, 62 | }, 63 | { 64 | name: "singlequotednumber", 65 | expected: 1, 66 | }, 67 | { 68 | name: "doublequotednumber", 69 | expected: 1, 70 | }, 71 | { 72 | name: "string", 73 | expected: "abc", 74 | }, 75 | { 76 | name: "singlequotedstring", 77 | expected: "abc", 78 | }, 79 | { 80 | name: "doublequotedstring", 81 | expected: "abc", 82 | }, 83 | { 84 | name: "array", 85 | expected: []any{1, 2, 3}, 86 | }, 87 | { 88 | name: "singlequotedarray", 89 | expected: []any{1, 2, 3}, 90 | }, 91 | { 92 | name: "doublequotedarray", 93 | expected: []any{1, 2, 3}, 94 | }, 95 | { 96 | name: "object", 97 | expected: vars.VarSet{"a": 1, "b": 2, "c": "3", "d": []any{4, "5"}}, 98 | }, 99 | { 100 | name: "singlequotedobject", 101 | expected: vars.VarSet{"a": 1, "b": 2, "c": "3", "d": []any{4, "5"}}, 102 | }, 103 | { 104 | name: "doublequotedobject", 105 | expected: vars.VarSet{"a": 1, "b": 2, "c": "3", "d": []any{4, "5"}}, 106 | }, 107 | { 108 | name: "string_output", 109 | expected: "a", 110 | }, 111 | { 112 | name: "number_output", 113 | expected: 1, 114 | }, 115 | { 116 | name: "array_output", 117 | expected: []any{1, 2, 3}, 118 | }, 119 | { 120 | name: "object_output", 121 | expected: vars.VarSet{"a": 1, "b": 2, "c": "3", "d": []any{4, "5"}}, 122 | }, 123 | } 124 | 125 | for _, tt := range tests { 126 | t.Run(tt.name, func(t *testing.T) { 127 | assert.DeepEqual(t, c.Vars[tt.name], tt.expected) 128 | }) 129 | } 130 | }) 131 | }) 132 | } 133 | -------------------------------------------------------------------------------- /tests/integration/complex-types/testdata/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | description: test corral 3 | commands: 4 | - module: module 5 | - command: /app/setvariables.sh 6 | node_pools: 7 | - all 8 | - module: variables 9 | variables: 10 | string_output: 11 | type: string 12 | readOnly: true 13 | number_output: 14 | type: number 15 | readOnly: true 16 | array_output: 17 | type: array 18 | readOnly: true 19 | object_output: 20 | type: object 21 | readOnly: true -------------------------------------------------------------------------------- /tests/integration/complex-types/testdata/overlay/app/setvariables.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo corral_set number=1 4 | echo corral_set singlequotednumber='1' 5 | echo corral_set doublequotednumber="1" 6 | 7 | echo corral_set string=abc 8 | echo corral_set singlequotedstring='abc' 9 | echo corral_set doublequotedstring="abc" 10 | 11 | echo corral_set array=[1,2,3] 12 | echo corral_set singlequotedarray='[1,2,3]' 13 | echo corral_set doublequotedarray="[1,2,3]" 14 | 15 | echo corral_set object=\{\"a\":1,\"b\":2.0,\"c\":\"3\",\"d\":[4,\"5\"]\} 16 | echo corral_set singlequotedobject='{"a":1,"b":2.0, "c":"3", "d":[4, "5"]}' 17 | echo corral_set doublequotedobject="{\"a\":1,\"b\":2.0, \"c\":\"3\", \"d\":[4, \"5\"]}" 18 | 19 | echo corral_set string_output="a" 20 | echo corral_set number_output=1 21 | echo corral_set array_output="[1,2,3]" 22 | echo corral_set object_output='{"a":1,"b":2.0, "c":"3", "d":[4, "5"]}' 23 | -------------------------------------------------------------------------------- /tests/integration/complex-types/testdata/terraform/module/corral.tf: -------------------------------------------------------------------------------- 1 | variable "corral_name" {} 2 | variable "corral_public_key" {} 3 | 4 | output "corral_node_pools" { 5 | value = { 6 | all = [ 7 | for n in docker_container.node : { 8 | name = n.name 9 | user = "corral" 10 | address = "127.0.0.1:${n.ports[0].external}" 11 | } 12 | ] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /tests/integration/complex-types/testdata/terraform/module/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | docker = { 4 | source = "kreuzwerker/docker" 5 | version = "~> 2.13.0" 6 | } 7 | } 8 | } 9 | 10 | provider "docker" {} 11 | 12 | resource "docker_volume" "data" { 13 | count = 1 14 | name = "${var.corral_name}-node-${count.index}" 15 | } 16 | 17 | resource "docker_container" "node" { 18 | count = 1 19 | image = "lscr.io/linuxserver/openssh-server" 20 | name = "${var.corral_name}-node-${count.index}" 21 | 22 | ports { 23 | internal = 2222 24 | } 25 | 26 | env = [ 27 | "PUBLIC_KEY=${var.corral_public_key}", 28 | "USER_NAME=corral", 29 | "USER_PASSWORD=corral", 30 | "SUDO_ACCESS=true", 31 | "PASSWORD_ACCESS=true", 32 | ] 33 | 34 | volumes { 35 | container_path = "/app" 36 | volume_name = docker_volume.data[count.index].name 37 | } 38 | } -------------------------------------------------------------------------------- /tests/integration/complex-types/testdata/terraform/variables/corral.tf: -------------------------------------------------------------------------------- 1 | variable "string_output" { 2 | type = string 3 | } 4 | 5 | variable "number_output" { 6 | type = number 7 | } 8 | 9 | variable "array_output" { 10 | type = list(any) 11 | } 12 | 13 | variable "object_output" { 14 | type = any 15 | } 16 | 17 | output "string_output" { 18 | value = var.string_output 19 | } 20 | 21 | output "number_output" { 22 | value = var.number_output 23 | } 24 | 25 | output "array_output" { 26 | value = var.array_output 27 | } 28 | 29 | output "object_output" { 30 | value = var.object_output 31 | } -------------------------------------------------------------------------------- /tests/integration/complex-types/testdata/terraform/variables/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rancherlabs/corral/cab06894ef96a71f7becccee8ebcb2a762a3eeca/tests/integration/complex-types/testdata/terraform/variables/main.tf -------------------------------------------------------------------------------- /tests/integration/template/template_suite_test.go: -------------------------------------------------------------------------------- 1 | package template_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestTemplate(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Template Suite") 13 | } 14 | -------------------------------------------------------------------------------- /tests/integration/template/template_test.go: -------------------------------------------------------------------------------- 1 | package template_test 2 | 3 | import ( 4 | "os" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | "github.com/rancherlabs/corral/pkg/config" 9 | _package "github.com/rancherlabs/corral/pkg/package" 10 | ) 11 | 12 | var _ = Describe("Template and package tests", func() { 13 | BeforeEach(func() { 14 | config.InitializeRootPath(GinkgoT().TempDir()) 15 | }) 16 | When("a local package does not exist", func() { 17 | It("should report an error", func() { 18 | Expect(_package.Template("test", "", "./doesnotexist")).Should(MatchError(os.ErrNotExist)) 19 | }) 20 | }) 21 | When("a remote package does not exist", func() { 22 | It("should report an error", func() { 23 | Expect(_package.Template("test", "", "doesnotexist:latest")).ToNot(BeNil()) 24 | }) 25 | }) 26 | When("the templates are valid", func() { 27 | It("should create successfully", func() { 28 | Expect(_package.Template("test", "", "testdata/template/template1", "testdata/template/template2")).To(BeNil()) 29 | pkg, err := _package.LoadPackage("test") 30 | Expect(err).To(BeNil()) 31 | Expect(pkg.Description).To(Equal("template1 input\ntemplate2 input")) 32 | }) 33 | }) 34 | When("description is not empty", func() { 35 | It("should utilize the description", func() { 36 | Expect(_package.Template("test", "test description", "testdata/template/template1", "testdata/template/template2")).To(BeNil()) 37 | pkg, err := _package.LoadPackage("test") 38 | Expect(err).To(BeNil()) 39 | Expect(pkg.Description).To(Equal("test description")) 40 | }) 41 | }) 42 | }) 43 | -------------------------------------------------------------------------------- /tests/integration/template/test/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | description: test description 3 | commands: 4 | - command: /app/template1.sh 5 | node_pools: 6 | - all 7 | - command: /app/template2.sh 8 | node_pools: 9 | - all 10 | variables: 11 | var1: 12 | default: t1 13 | description: template variable 1 14 | type: string 15 | var2: 16 | default: t2 17 | description: template variable 2 18 | type: string 19 | -------------------------------------------------------------------------------- /tests/integration/template/test/overlay/app/template1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | env 4 | 5 | echo "corral_set var1_out=${CORRAL_var1}" 6 | -------------------------------------------------------------------------------- /tests/integration/template/test/overlay/app/template2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | env 4 | 5 | echo "corral_set var1_out=${CORRAL_var1}" 6 | -------------------------------------------------------------------------------- /tests/integration/template/testdata/template/config.yaml: -------------------------------------------------------------------------------- 1 | name: template 2 | description: > 3 | template test 4 | packages: 5 | - template1 6 | - template2 -------------------------------------------------------------------------------- /tests/integration/template/testdata/template/template1/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: template1 input 2 | description: template1 input 3 | commands: 4 | - command: /app/template1.sh 5 | node_pools: 6 | - all 7 | variables: 8 | var1: 9 | default: t1 10 | description: template variable 1 11 | type: string 12 | -------------------------------------------------------------------------------- /tests/integration/template/testdata/template/template1/overlay/app/template1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | env 4 | 5 | echo "corral_set var1_out=${CORRAL_var1}" 6 | -------------------------------------------------------------------------------- /tests/integration/template/testdata/template/template2/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: template2 input 2 | description: template2 input 3 | commands: 4 | - command: /app/template2.sh 5 | node_pools: 6 | - all 7 | variables: 8 | var2: 9 | default: t2 10 | description: template variable 2 11 | type: string 12 | -------------------------------------------------------------------------------- /tests/integration/template/testdata/template/template2/overlay/app/template2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | env 4 | 5 | echo "corral_set var1_out=${CORRAL_var1}" 6 | -------------------------------------------------------------------------------- /tests/integration/validate/testdata/no_module/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: no_module 2 | description: Test no module 3 | commands: 4 | - module: dummy -------------------------------------------------------------------------------- /tests/integration/validate/testdata/no_overlay/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: no_overlay 2 | description: Test no overlay 3 | commands: 4 | - command: test.sh 5 | node_pools: 6 | - dummy -------------------------------------------------------------------------------- /tests/integration/validate/testdata/valid/manifest.yaml: -------------------------------------------------------------------------------- 1 | name: valid 2 | description: minimally valid corral package 3 | commands: 4 | - module: module 5 | - command: test.sh 6 | node_pools: 7 | - test -------------------------------------------------------------------------------- /tests/integration/validate/testdata/valid/overlay/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Hello World!" -------------------------------------------------------------------------------- /tests/integration/validate/testdata/valid/terraform/module/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rancherlabs/corral/cab06894ef96a71f7becccee8ebcb2a762a3eeca/tests/integration/validate/testdata/valid/terraform/module/main.tf -------------------------------------------------------------------------------- /tests/integration/validate/validate_suite_test.go: -------------------------------------------------------------------------------- 1 | package validate_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestValidate(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Validate Suite") 13 | } 14 | -------------------------------------------------------------------------------- /tests/integration/validate/validate_test.go: -------------------------------------------------------------------------------- 1 | package validate_test 2 | 3 | import ( 4 | "os" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | "github.com/rancherlabs/corral/pkg/config" 9 | _package "github.com/rancherlabs/corral/pkg/package" 10 | ) 11 | 12 | var _ = Describe("Validate packages", func() { 13 | BeforeEach(func() { 14 | config.InitializeRootPath(GinkgoT().TempDir()) 15 | }) 16 | When("the package does not have a manifest", func() { 17 | It("should not be validated", func() { 18 | Expect(_package.Validate("./testdata/no_manifest")).Should(MatchError(os.ErrNotExist)) 19 | }) 20 | }) 21 | When("the package does not have an overlay folder", func() { 22 | It("should not be validated", func() { 23 | Expect(_package.Validate("./testdata/no_overlay")).Should(MatchError(_package.ErrOverlayNotFound)) 24 | }) 25 | }) 26 | When("the package uses a module that is not present", func() { 27 | It("should not be validated", func() { 28 | Expect(_package.Validate("./testdata/no_manifest")).Should(MatchError(os.ErrNotExist)) 29 | }) 30 | }) 31 | When("the package is valid", func() { 32 | It("should be validated", func() { 33 | Expect(_package.Validate("./testdata/valid")).To(BeNil()) 34 | }) 35 | }) 36 | }) 37 | --------------------------------------------------------------------------------