├── .golangci.yml ├── hack ├── zstDictionary ├── docgen.go ├── gen-cert-chain.sh └── fake-eula.txt ├── .dockerignore ├── .gitignore ├── examples ├── whoami │ ├── README.md │ └── whoami.yaml ├── whoami-vars │ ├── k3p.yaml │ ├── README.md │ └── whoami.yaml ├── docker │ ├── whoami.yaml │ └── README.md ├── helm-charts │ ├── k3p.yaml │ └── README.md └── ha │ ├── whoami.yaml │ └── README.md ├── pkg ├── version │ └── version.go ├── images │ ├── image_downloader.go │ ├── save_images.go │ ├── build_registry.go │ ├── util.go │ └── registry │ │ └── templates.go ├── cmd │ ├── util.go │ ├── version.go │ ├── cache.go │ ├── root.go │ ├── uninstall.go │ ├── completion.go │ ├── token.go │ ├── nodes.go │ └── build.go ├── types │ ├── manifest_parser.go │ ├── util.go │ ├── image_downloader.go │ ├── registry_options.go │ ├── artifact.go │ ├── cluster.go │ ├── manifest.go │ ├── package.go │ ├── builder.go │ ├── node.go │ ├── package_meta.go │ ├── docker.go │ ├── constants.go │ └── installer.go ├── cluster │ ├── node │ │ ├── util.go │ │ ├── mock.go │ │ ├── local.go │ │ └── remote.go │ ├── kubernetes │ │ └── client.go │ └── manager.go ├── install │ ├── installer_test.go │ └── installer.go ├── build │ ├── package │ │ └── v1 │ │ │ ├── mock.go │ │ │ └── archive.go │ └── k3s_components.go ├── parser │ ├── base_parser.go │ ├── obj_parse.go │ └── helm.go ├── util │ ├── util_test.go │ └── ip_util.go ├── cache │ ├── download_test.go │ └── download.go └── log │ └── log.go ├── cmd └── k3p │ └── main.go ├── Dockerfile ├── doc ├── k3p_cache_clean.md ├── k3p_version.md ├── k3p_cache.md ├── k3p_uninstall.md ├── k3p_token.md ├── k3p_token_generate.md ├── k3p_inspect.md ├── k3p_token_get.md ├── k3p.md ├── k3p_completion.md ├── k3p_node.md ├── k3p_node_add.md ├── k3p_node_remove.md ├── k3p_build.md └── k3p_install.md ├── .github └── workflows │ ├── release.yaml │ └── ci.yml ├── go.mod ├── Makefile └── README.md /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | skip-dirs: 3 | - pkg/cmd -------------------------------------------------------------------------------- /hack/zstDictionary: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzimmer/k3p/HEAD/hack/zstDictionary -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | .golangci.yml 3 | examples/ 4 | doc/ 5 | .github/ 6 | README.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | vendor/ 3 | *.tar 4 | *.tgz 5 | *.tar.gz 6 | *.tar.zst 7 | *.coverprofile 8 | *.dat 9 | *.run 10 | *kubeconfig.yaml* 11 | tls/ -------------------------------------------------------------------------------- /examples/whoami/README.md: -------------------------------------------------------------------------------- 1 | # Simple Example 2 | 3 | This directory contains one of the simplest examples of building a package. 4 | There is no dynamic configuration, just a single manifest. This is the same example as on the [main page](../../README.md). -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | // K3pVersion is populated with the version of this binary at compilation. 4 | var K3pVersion string 5 | 6 | // K3pCommit is populated with the commit of this binary at compilation 7 | var K3pCommit string 8 | -------------------------------------------------------------------------------- /cmd/k3p/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/tinyzimmer/k3p/pkg/cmd" 7 | "github.com/tinyzimmer/k3p/pkg/log" 8 | ) 9 | 10 | func main() { 11 | if err := cmd.GetRootCommand().Execute(); err != nil { 12 | log.Error(err) 13 | os.Exit(1) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/whoami-vars/k3p.yaml: -------------------------------------------------------------------------------- 1 | variables: 2 | - name: dnsName 3 | default: "localhost" 4 | - name: traefikDisabled 5 | default: "false" 6 | --- 7 | serverConfig: 8 | disable: 9 | - local-storage 10 | - metrics-server 11 | {{- if eq .Vars.traefikDisabled "true" }} 12 | - traefik 13 | {{ end }} 14 | -------------------------------------------------------------------------------- /pkg/images/image_downloader.go: -------------------------------------------------------------------------------- 1 | package images 2 | 3 | import "github.com/tinyzimmer/k3p/pkg/types" 4 | 5 | // NewImageDownloader returns a new interface for downloading and exporting container 6 | // images. 7 | func NewImageDownloader() types.ImageDownloader { 8 | return &dockerImageDownloader{} 9 | } 10 | 11 | type dockerImageDownloader struct{} 12 | -------------------------------------------------------------------------------- /pkg/cmd/util.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | func completeStringOpts(opts []string) func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { 8 | return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { 9 | return opts, cobra.ShellCompDirectiveDefault 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /pkg/cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | 8 | "github.com/tinyzimmer/k3p/pkg/version" 9 | ) 10 | 11 | func init() { 12 | rootCmd.AddCommand(versionCmd) 13 | } 14 | 15 | var versionCmd = &cobra.Command{ 16 | Use: "version", 17 | Short: "Display version information for k3p", 18 | Run: func(cmd *cobra.Command, args []string) { 19 | fmt.Println("K3P Version:", version.K3pVersion) 20 | fmt.Println("K3P GitCommit:", version.K3pCommit) 21 | }, 22 | } 23 | -------------------------------------------------------------------------------- /pkg/types/manifest_parser.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // ManifestParser is an interface for extracting a list of images/manifests from a directory. 4 | type ManifestParser interface { 5 | // ParseImages should traverse the configured directories and search for container images 6 | // to download. 7 | ParseImages() ([]string, error) 8 | // ParseManifests should traverse the configured directories and produce artifacts for 9 | // every kubernetes manifest it finds. 10 | ParseManifests() ([]*Artifact, error) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/types/util.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "bytes" 5 | "text/template" 6 | 7 | "github.com/Masterminds/sprig" 8 | ) 9 | 10 | func render(body []byte, vars map[string]string) ([]byte, error) { 11 | tmpl, err := template.New("").Funcs(sprig.TxtFuncMap()).Parse(string(body)) 12 | if err != nil { 13 | return nil, err 14 | } 15 | var out bytes.Buffer 16 | if err := tmpl.Execute(&out, map[string]interface{}{ 17 | "Vars": vars, 18 | }); err != nil { 19 | return nil, err 20 | } 21 | return out.Bytes(), nil 22 | } 23 | -------------------------------------------------------------------------------- /pkg/cmd/cache.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/tinyzimmer/k3p/pkg/cache" 7 | ) 8 | 9 | func init() { 10 | cacheCmd.AddCommand(cacheCleanCmd) 11 | rootCmd.AddCommand(cacheCmd) 12 | } 13 | 14 | var cacheCmd = &cobra.Command{ 15 | Use: "cache", 16 | Short: "Cache management options", 17 | } 18 | 19 | var cacheCleanCmd = &cobra.Command{ 20 | Use: "clean", 21 | Short: "Wipe the local artifact cache", 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | return cache.DefaultCache.Clean() 24 | }, 25 | } 26 | -------------------------------------------------------------------------------- /pkg/cluster/node/util.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/tinyzimmer/k3p/pkg/types" 8 | ) 9 | 10 | func redactSecrets(logLine string, secrets []string) string { 11 | for _, secret := range secrets { 12 | logLine = strings.Replace(logLine, secret, "", -1) 13 | } 14 | return logLine 15 | } 16 | 17 | func buildCmdFromExecOpts(opts *types.ExecuteOptions) string { 18 | var cmd string 19 | for k, v := range opts.Env { 20 | cmd = cmd + fmt.Sprintf("%s=%q ", k, v) 21 | } 22 | cmd = cmd + "sudo -E " + opts.Command 23 | return cmd 24 | } 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15 as builder 2 | 3 | RUN mkdir -p /workspace \ 4 | && apt-get update \ 5 | && apt-get install -y upx 6 | 7 | WORKDIR /workspace 8 | 9 | COPY go.mod /workspace/go.mod 10 | COPY go.sum /workspace/go.sum 11 | 12 | RUN go mod download 13 | 14 | COPY . /workspace/ 15 | 16 | RUN make 17 | 18 | # Make a small alpine image 19 | FROM alpine:latest 20 | 21 | # Create a default working directory 22 | RUN mkdir -p /manifests 23 | WORKDIR /manifests 24 | 25 | # Copy the binary over 26 | COPY --from=builder /workspace/dist/k3p /usr/local/bin/k3p 27 | ENTRYPOINT [ "/usr/local/bin/k3p" ] 28 | -------------------------------------------------------------------------------- /doc/k3p_cache_clean.md: -------------------------------------------------------------------------------- 1 | ## k3p cache clean 2 | 3 | Wipe the local artifact cache 4 | 5 | ``` 6 | k3p cache clean [flags] 7 | ``` 8 | 9 | ### Options 10 | 11 | ``` 12 | -h, --help help for clean 13 | ``` 14 | 15 | ### Options inherited from parent commands 16 | 17 | ``` 18 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 19 | --tmp-dir string Override the default tmp directory (default "/tmp") 20 | -v, --verbose Enable verbose logging 21 | ``` 22 | 23 | ### SEE ALSO 24 | 25 | * [k3p cache](k3p_cache.md) - Cache management options 26 | 27 | -------------------------------------------------------------------------------- /doc/k3p_version.md: -------------------------------------------------------------------------------- 1 | ## k3p version 2 | 3 | Display version information for k3p 4 | 5 | ``` 6 | k3p version [flags] 7 | ``` 8 | 9 | ### Options 10 | 11 | ``` 12 | -h, --help help for version 13 | ``` 14 | 15 | ### Options inherited from parent commands 16 | 17 | ``` 18 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 19 | --tmp-dir string Override the default tmp directory (default "/tmp") 20 | -v, --verbose Enable verbose logging 21 | ``` 22 | 23 | ### SEE ALSO 24 | 25 | * [k3p](k3p.md) - k3p is a k3s packaging and delivery utility 26 | 27 | -------------------------------------------------------------------------------- /doc/k3p_cache.md: -------------------------------------------------------------------------------- 1 | ## k3p cache 2 | 3 | Cache management options 4 | 5 | ### Options 6 | 7 | ``` 8 | -h, --help help for cache 9 | ``` 10 | 11 | ### Options inherited from parent commands 12 | 13 | ``` 14 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 15 | --tmp-dir string Override the default tmp directory (default "/tmp") 16 | -v, --verbose Enable verbose logging 17 | ``` 18 | 19 | ### SEE ALSO 20 | 21 | * [k3p](k3p.md) - k3p is a k3s packaging and delivery utility 22 | * [k3p cache clean](k3p_cache_clean.md) - Wipe the local artifact cache 23 | 24 | -------------------------------------------------------------------------------- /doc/k3p_uninstall.md: -------------------------------------------------------------------------------- 1 | ## k3p uninstall 2 | 3 | Uninstall a k3p package (currently only for docker) 4 | 5 | ``` 6 | k3p uninstall [flags] 7 | ``` 8 | 9 | ### Options 10 | 11 | ``` 12 | -h, --help help for uninstall 13 | ``` 14 | 15 | ### Options inherited from parent commands 16 | 17 | ``` 18 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 19 | --tmp-dir string Override the default tmp directory (default "/tmp") 20 | -v, --verbose Enable verbose logging 21 | ``` 22 | 23 | ### SEE ALSO 24 | 25 | * [k3p](k3p.md) - k3p is a k3s packaging and delivery utility 26 | 27 | -------------------------------------------------------------------------------- /pkg/images/save_images.go: -------------------------------------------------------------------------------- 1 | package images 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/tinyzimmer/k3p/pkg/log" 8 | "github.com/tinyzimmer/k3p/pkg/types" 9 | ) 10 | 11 | func (d *dockerImageDownloader) SaveImages(images []string, arch string, pullPolicy types.PullPolicy) (io.ReadCloser, error) { 12 | cli, err := getDockerClient() 13 | if err != nil { 14 | return nil, err 15 | } 16 | defer cli.Close() 17 | 18 | images = sanitizeImageNameSlice(images) 19 | for _, image := range images { 20 | if err := ensureImagePulled(cli, image, arch, pullPolicy); err != nil { 21 | return nil, err 22 | } 23 | } 24 | 25 | log.Debug("Saving images:", images) 26 | return cli.ImageSave(context.TODO(), images) 27 | } 28 | -------------------------------------------------------------------------------- /doc/k3p_token.md: -------------------------------------------------------------------------------- 1 | ## k3p token 2 | 3 | Token retrieval and generation commands 4 | 5 | ### Options 6 | 7 | ``` 8 | -h, --help help for token 9 | ``` 10 | 11 | ### Options inherited from parent commands 12 | 13 | ``` 14 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 15 | --tmp-dir string Override the default tmp directory (default "/tmp") 16 | -v, --verbose Enable verbose logging 17 | ``` 18 | 19 | ### SEE ALSO 20 | 21 | * [k3p](k3p.md) - k3p is a k3s packaging and delivery utility 22 | * [k3p token generate](k3p_token_generate.md) - Generates a token that can be used for initializing HA installations 23 | * [k3p token get](k3p_token_get.md) - Retrieve a k3s token 24 | 25 | -------------------------------------------------------------------------------- /doc/k3p_token_generate.md: -------------------------------------------------------------------------------- 1 | ## k3p token generate 2 | 3 | Generates a token that can be used for initializing HA installations 4 | 5 | ``` 6 | k3p token generate [flags] 7 | ``` 8 | 9 | ### Options 10 | 11 | ``` 12 | -h, --help help for generate 13 | -l, --length int The length of the token to generate (default 128) 14 | ``` 15 | 16 | ### Options inherited from parent commands 17 | 18 | ``` 19 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 20 | --tmp-dir string Override the default tmp directory (default "/tmp") 21 | -v, --verbose Enable verbose logging 22 | ``` 23 | 24 | ### SEE ALSO 25 | 26 | * [k3p token](k3p_token.md) - Token retrieval and generation commands 27 | 28 | -------------------------------------------------------------------------------- /doc/k3p_inspect.md: -------------------------------------------------------------------------------- 1 | ## k3p inspect 2 | 3 | Inspect the given package 4 | 5 | ``` 6 | k3p inspect PACKAGE [flags] 7 | ``` 8 | 9 | ### Options 10 | 11 | ``` 12 | -c, --config string Dump the contents of the specified config file 13 | -D, --details Show additional details on package content 14 | -h, --help help for inspect 15 | -m, --manifest string Dump the contents of the specified manifest 16 | ``` 17 | 18 | ### Options inherited from parent commands 19 | 20 | ``` 21 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 22 | --tmp-dir string Override the default tmp directory (default "/tmp") 23 | -v, --verbose Enable verbose logging 24 | ``` 25 | 26 | ### SEE ALSO 27 | 28 | * [k3p](k3p.md) - k3p is a k3s packaging and delivery utility 29 | 30 | -------------------------------------------------------------------------------- /pkg/types/image_downloader.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import "io" 4 | 5 | // ImageDownloader is an interface for pulling OCI container images and exporting 6 | // them to tar archives or deployable registries. It can be implemented by different 7 | // runtimes such as docker, containerd, podman, etc. 8 | type ImageDownloader interface { 9 | // SaveImages will return a reader containing the contents of the exported 10 | // images provided as arguments. 11 | SaveImages(images []string, arch string, pullPolicy PullPolicy) (io.ReadCloser, error) 12 | // BuildRegistry will build a container registry with the given images and return a 13 | // a reader to a container image holding the backed up contents. It will be unpacked into 14 | // a running registry with auto-generated TLS at installation time. 15 | BuildRegistry(*BuildRegistryOptions) (io.ReadCloser, error) 16 | } 17 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | # Sequence of patterns matched against refs/tags 4 | tags: 5 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 6 | 7 | name: Create Release 8 | 9 | jobs: 10 | release: 11 | name: Create Release 12 | runs-on: ubuntu-latest 13 | steps: 14 | 15 | - name: Checkout Code 16 | uses: actions/checkout@v2 17 | 18 | - name: Set up Go 1.15 19 | uses: actions/setup-go@v1 20 | with: 21 | go-version: 1.15 22 | id: go 23 | 24 | - name: Install upx 25 | run: sudo apt-get install -y upx 26 | 27 | - name: Build Release 28 | run: COMPRESSION=9 make dist 29 | 30 | - name: Publish Release Artifacts 31 | uses: softprops/action-gh-release@v1 32 | env: 33 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 34 | with: 35 | files: | 36 | dist/* -------------------------------------------------------------------------------- /doc/k3p_token_get.md: -------------------------------------------------------------------------------- 1 | ## k3p token get 2 | 3 | Retrieve a k3s token 4 | 5 | ### Synopsis 6 | 7 | 8 | Retrieves the token for joining either a new "agent" or "server" to the cluster. 9 | 10 | The "agent" token can be retrieved from any of the server instances, while the "server" token 11 | can only be retrieved on the server where "k3p install" was run with "--init-ha". 12 | 13 | 14 | ``` 15 | k3p token get TOKEN_TYPE [flags] 16 | ``` 17 | 18 | ### Options 19 | 20 | ``` 21 | -h, --help help for get 22 | ``` 23 | 24 | ### Options inherited from parent commands 25 | 26 | ``` 27 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 28 | --tmp-dir string Override the default tmp directory (default "/tmp") 29 | -v, --verbose Enable verbose logging 30 | ``` 31 | 32 | ### SEE ALSO 33 | 34 | * [k3p token](k3p_token.md) - Token retrieval and generation commands 35 | 36 | -------------------------------------------------------------------------------- /pkg/install/installer_test.go: -------------------------------------------------------------------------------- 1 | package install 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | 9 | v1 "github.com/tinyzimmer/k3p/pkg/build/package/v1" 10 | "github.com/tinyzimmer/k3p/pkg/cluster/node" 11 | "github.com/tinyzimmer/k3p/pkg/log" 12 | "github.com/tinyzimmer/k3p/pkg/types" 13 | ) 14 | 15 | func TestUtils(t *testing.T) { 16 | log.LogWriter = GinkgoWriter 17 | RegisterFailHandler(Fail) 18 | RunSpecs(t, "Installer Suite") 19 | } 20 | 21 | var _ = Describe("Installer", func() { 22 | var ( 23 | err error 24 | opts types.InstallOptions 25 | ) 26 | 27 | target := node.Mock() 28 | defer target.Close() 29 | 30 | JustBeforeEach(func() { 31 | err = New().Install(target, v1.Mock(), &opts) 32 | }) 33 | 34 | Context("With no error conditions present", func() { 35 | It("Should succeed", func() { 36 | Expect(err).ToNot(HaveOccurred()) 37 | }) 38 | }) 39 | 40 | // TODO: More tests 41 | }) 42 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tinyzimmer/k3p 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/Masterminds/semver v1.5.0 // indirect 7 | github.com/Masterminds/sprig v2.22.0+incompatible 8 | github.com/Microsoft/go-winio v0.4.15 // indirect 9 | github.com/bramvdbogaerde/go-scp v0.0.0-20200820121624-ded9ee94aef5 10 | github.com/containerd/containerd v1.4.2 // indirect 11 | github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible 12 | github.com/docker/go-connections v0.4.0 13 | github.com/gorilla/mux v1.8.0 // indirect 14 | github.com/huandu/xstrings v1.3.2 // indirect 15 | github.com/klauspost/compress v1.11.3 16 | github.com/mitchellh/go-ps v1.0.0 17 | github.com/onsi/ginkgo v1.14.2 18 | github.com/onsi/gomega v1.10.3 19 | github.com/spf13/cobra v1.1.1 20 | golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 21 | golang.org/x/sys v0.0.0-20201218084310-7d0127a74742 // indirect 22 | gopkg.in/yaml.v2 v2.4.0 23 | helm.sh/helm v2.17.0+incompatible 24 | helm.sh/helm/v3 v3.4.2 25 | k8s.io/api v0.19.4 26 | k8s.io/apimachinery v0.19.4 27 | k8s.io/client-go v0.19.4 28 | k8s.io/helm v2.17.0+incompatible // indirect 29 | ) 30 | -------------------------------------------------------------------------------- /pkg/build/package/v1/mock.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "io/ioutil" 5 | "strings" 6 | 7 | "github.com/tinyzimmer/k3p/pkg/types" 8 | ) 9 | 10 | var mockArtifacts = []*types.Artifact{ 11 | { 12 | Type: types.ArtifactBin, 13 | Name: "k3s", 14 | Body: ioutil.NopCloser(strings.NewReader("test")), 15 | Size: 4, 16 | }, 17 | { 18 | Type: types.ArtifactImages, 19 | Name: "k3s-airgap-images.tar", 20 | Body: ioutil.NopCloser(strings.NewReader("test")), 21 | Size: 4, 22 | }, 23 | { 24 | Type: types.ArtifactScript, 25 | Name: "install.sh", 26 | Body: ioutil.NopCloser(strings.NewReader("test")), 27 | Size: 4, 28 | }, 29 | { 30 | Type: types.ArtifactManifest, 31 | Name: "manifest.yaml", 32 | Body: ioutil.NopCloser(strings.NewReader("test")), 33 | Size: 4, 34 | }, 35 | } 36 | 37 | // Mock returns a fake package. 38 | func Mock() types.Package { 39 | tmpDir, err := ioutil.TempDir("", "") 40 | if err != nil { 41 | panic(err) 42 | } 43 | writer := New(tmpDir) 44 | for _, artifact := range mockArtifacts { 45 | if err := writer.Put(artifact); err != nil { 46 | panic(err) 47 | } 48 | } 49 | return writer 50 | } 51 | -------------------------------------------------------------------------------- /examples/whoami/whoami.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: whoami 6 | namespace: default 7 | labels: 8 | app: whoami 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app: whoami 14 | template: 15 | metadata: 16 | labels: 17 | app: whoami 18 | annotations: 19 | spec: 20 | containers: 21 | - name: whoami 22 | image: traefik/whoami:latest 23 | imagePullPolicy: IfNotPresent 24 | ports: 25 | - name: http 26 | containerPort: 80 27 | --- 28 | apiVersion: v1 29 | kind: Service 30 | metadata: 31 | name: whoami 32 | namespace: default 33 | labels: 34 | app: whoami 35 | spec: 36 | type: ClusterIP 37 | ports: 38 | - name: http 39 | port: 80 40 | targetPort: 80 41 | selector: 42 | app: whoami 43 | --- 44 | kind: Ingress 45 | apiVersion: extensions/v1beta1 46 | metadata: 47 | name: whoami 48 | namespace: default 49 | spec: 50 | rules: 51 | - host: localhost 52 | http: 53 | paths: 54 | - path: / 55 | backend: 56 | serviceName: whoami 57 | servicePort: 80 58 | -------------------------------------------------------------------------------- /examples/docker/whoami.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: whoami 6 | namespace: default 7 | labels: 8 | app: whoami 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app: whoami 14 | template: 15 | metadata: 16 | labels: 17 | app: whoami 18 | annotations: 19 | spec: 20 | containers: 21 | - name: whoami 22 | image: "traefik/whoami:latest" 23 | imagePullPolicy: IfNotPresent 24 | ports: 25 | - name: http 26 | containerPort: 80 27 | --- 28 | apiVersion: v1 29 | kind: Service 30 | metadata: 31 | name: whoami 32 | namespace: default 33 | labels: 34 | app: whoami 35 | spec: 36 | type: ClusterIP 37 | ports: 38 | - name: http 39 | port: 80 40 | targetPort: 80 41 | selector: 42 | app: whoami 43 | --- 44 | kind: Ingress 45 | apiVersion: extensions/v1beta1 46 | metadata: 47 | name: whoami 48 | namespace: default 49 | spec: 50 | rules: 51 | - host: localhost 52 | http: 53 | paths: 54 | - path: / 55 | backend: 56 | serviceName: whoami 57 | servicePort: 80 58 | -------------------------------------------------------------------------------- /doc/k3p.md: -------------------------------------------------------------------------------- 1 | ## k3p 2 | 3 | k3p is a k3s packaging and delivery utility 4 | 5 | ### Synopsis 6 | 7 | 8 | The k3p command provides an easy method for packaging a kubernetes environment into a distributable object. 9 | 10 | 11 | ### Options 12 | 13 | ``` 14 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 15 | -h, --help help for k3p 16 | --tmp-dir string Override the default tmp directory (default "/tmp") 17 | -v, --verbose Enable verbose logging 18 | ``` 19 | 20 | ### SEE ALSO 21 | 22 | * [k3p build](k3p_build.md) - Build a k3s distribution package 23 | * [k3p cache](k3p_cache.md) - Cache management options 24 | * [k3p completion](k3p_completion.md) - Generate completion script 25 | * [k3p inspect](k3p_inspect.md) - Inspect the given package 26 | * [k3p install](k3p_install.md) - Install the given package to the system 27 | * [k3p node](k3p_node.md) - Node management commands 28 | * [k3p token](k3p_token.md) - Token retrieval and generation commands 29 | * [k3p uninstall](k3p_uninstall.md) - Uninstall a k3p package (currently only for docker) 30 | * [k3p version](k3p_version.md) - Display version information for k3p 31 | 32 | -------------------------------------------------------------------------------- /examples/helm-charts/k3p.yaml: -------------------------------------------------------------------------------- 1 | variables: 2 | - name: enableMetrics 3 | default: "false" 4 | 5 | --- 6 | serverConfig: 7 | disable: traefik 8 | 9 | helmValues: 10 | 11 | kvdi: 12 | vdi: 13 | spec: 14 | auth: 15 | allowAnonymous: true 16 | app: 17 | auditLog: true 18 | {{ if eq .Vars.enableMetrics "true" }} 19 | metrics: 20 | prometheus: 21 | create: true 22 | grafana: 23 | enabled: true 24 | serviceMonitor: 25 | create: true 26 | labels: 27 | release: kube-prometheus-stack 28 | {{ end }} 29 | 30 | kube-prometheus-stack: 31 | # JUST installs the operator 32 | defaultRules: 33 | create: false 34 | prometheus: 35 | enabled: false 36 | alertmanager: 37 | enabled: false 38 | grafana: 39 | enabled: false 40 | nodeExporter: 41 | enabled: false 42 | kubelet: 43 | enabled: false 44 | kubeStateMetrics: 45 | enabled: false 46 | kubeScheduler: 47 | enabled: false 48 | kubeProxy: 49 | enabled: false 50 | kubeEtcd: 51 | enabled: false 52 | kubeDns: 53 | enabled: false 54 | kubeControllerManager: 55 | enabled: false 56 | kubeApiServer: 57 | enabled: false 58 | coreDns: 59 | enabled: false 60 | -------------------------------------------------------------------------------- /examples/whoami-vars/README.md: -------------------------------------------------------------------------------- 1 | # Example with Variables 2 | 3 | This example extends on the simpler [whoami](../whoami) example, except it utilizes 4 | the variable and templating functionality for accepting user input at installation. 5 | 6 | To build the example: 7 | 8 | ```bash 9 | # k3p build will use k3p.yaml automatically if it exists in the current working directory. 10 | # otherwise you can specify the path to one with the --config flag. 11 | $ k3p build 12 | # ... 13 | # ... 14 | ``` 15 | 16 | When the user goes to install the package, they will be prompted for input, or can alternatively use the `--set` flag to `install`. 17 | 18 | ```bash 19 | $ k3p install package.tar --docker 20 | 2020/12/13 20:08:47 [INFO] Loading the archive 21 | 2020/12/13 20:08:47 [INFO] Creating docker network vibrant_leakey 22 | 2020/12/13 20:08:48 [INFO] Creating docker volume vibrant-leakey-server-0 23 | Please provide a value for dnsName [localhost]: 24 | Please provide a value for traefikDisabled [false]: 25 | # ... 26 | # ... 27 | ``` 28 | 29 | The variable functionality is limited to strings and requires default values be set. For complex templating it is better to 30 | embed a helm chart and use the variables for simple substitutions on the values passed to that chart. You can see an example of this 31 | in the [helm-charts example](../helm-charts). -------------------------------------------------------------------------------- /examples/whoami-vars/whoami.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: whoami 6 | namespace: default 7 | labels: 8 | app: whoami 9 | spec: 10 | type: {{ if eq .Vars.traefikDisabled "false" }}ClusterIP{{ else }}LoadBalancer{{ end }} 11 | ports: 12 | - name: http 13 | port: 80 14 | targetPort: 80 15 | selector: 16 | app: whoami 17 | --- 18 | apiVersion: apps/v1 19 | kind: Deployment 20 | metadata: 21 | name: whoami 22 | namespace: default 23 | labels: 24 | app: whoami 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | app: whoami 30 | template: 31 | metadata: 32 | labels: 33 | app: whoami 34 | annotations: 35 | spec: 36 | containers: 37 | - name: whoami 38 | image: "traefik/whoami:latest" 39 | imagePullPolicy: Never 40 | ports: 41 | - name: http 42 | containerPort: 80 43 | 44 | {{ if eq .Vars.traefikDisabled "false" }} 45 | --- 46 | kind: Ingress 47 | apiVersion: extensions/v1beta1 48 | metadata: 49 | name: whoami 50 | namespace: default 51 | spec: 52 | rules: 53 | - host: {{ .Vars.dnsName }} 54 | http: 55 | paths: 56 | - path: / 57 | backend: 58 | serviceName: whoami 59 | servicePort: 80 60 | {{ end }} 61 | -------------------------------------------------------------------------------- /hack/docgen.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "os/user" 7 | "path" 8 | "path/filepath" 9 | "strings" 10 | 11 | "github.com/spf13/cobra/doc" 12 | 13 | "github.com/tinyzimmer/k3p/pkg/cmd" 14 | "github.com/tinyzimmer/k3p/pkg/log" 15 | ) 16 | 17 | func main() { 18 | if err := genMarkdownDocs(); err != nil { 19 | log.Fatal(err) 20 | } 21 | } 22 | 23 | func genMarkdownDocs() error { 24 | u, err := user.Current() 25 | if err != nil { 26 | return err 27 | } 28 | username := u.Username 29 | 30 | tmpDir, err := ioutil.TempDir("", "") 31 | if err != nil { 32 | return err 33 | } 34 | defer os.RemoveAll(tmpDir) 35 | 36 | if err := doc.GenMarkdownTree(cmd.GetRootCommand(), tmpDir); err != nil { 37 | log.Fatal(err) 38 | } 39 | 40 | if err := os.MkdirAll("doc", 0755); err != nil { 41 | log.Fatal(err) 42 | } 43 | 44 | return filepath.Walk(tmpDir, func(file string, fileInfo os.FileInfo, lastErr error) error { 45 | if lastErr != nil { 46 | return lastErr 47 | } 48 | if fileInfo.IsDir() { 49 | return nil 50 | } 51 | data, err := ioutil.ReadFile(file) 52 | if err != nil { 53 | return err 54 | } 55 | sanitized := strings.Replace(string(data), username, "", -1) 56 | if err := ioutil.WriteFile(path.Join("doc", strings.TrimPrefix(file, tmpDir+"/")), []byte(sanitized), 0644); err != nil { 57 | return err 58 | } 59 | return nil 60 | }) 61 | 62 | } 63 | -------------------------------------------------------------------------------- /pkg/cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/tinyzimmer/k3p/pkg/cache" 7 | "github.com/tinyzimmer/k3p/pkg/log" 8 | "github.com/tinyzimmer/k3p/pkg/util" 9 | ) 10 | 11 | var ( 12 | cacheDir string 13 | ) 14 | 15 | func init() { 16 | rootCmd.PersistentFlags().StringVar(&cacheDir, "cache-dir", cache.DefaultCache.CacheDir(), "Override the default location for cached k3s assets") 17 | rootCmd.PersistentFlags().StringVar(&util.TempDir, "tmp-dir", util.TempDir, "Override the default tmp directory") 18 | rootCmd.PersistentFlags().BoolVarP(&log.Verbose, "verbose", "v", false, "Enable verbose logging") 19 | } 20 | 21 | var rootCmd = &cobra.Command{ 22 | Use: "k3p", 23 | Short: "k3p is a k3s packaging and delivery utility", 24 | Long: ` 25 | The k3p command provides an easy method for packaging a kubernetes environment into a distributable object. 26 | `, 27 | SilenceUsage: true, 28 | DisableAutoGenTag: true, 29 | SilenceErrors: true, 30 | PersistentPreRun: func(cmd *cobra.Command, args []string) { 31 | if cacheDir != cache.DefaultCache.CacheDir() { 32 | log.Debugf("Setting cache dir to %q\n", cacheDir) 33 | cache.DefaultCache = cache.New(cacheDir) 34 | } else { 35 | log.Debugf("Default cache dir is %q\n", cache.DefaultCache.CacheDir()) 36 | } 37 | }, 38 | } 39 | 40 | // GetRootCommand returns the root k3p command 41 | func GetRootCommand() *cobra.Command { return rootCmd } 42 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | test: 12 | name: Run tests 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - uses: actions/checkout@v1 17 | 18 | - name: Set up Go 1.15 19 | uses: actions/setup-go@v1 20 | with: 21 | go-version: 1.15 22 | id: go 23 | 24 | - name: Check out code into the Go module directory 25 | uses: actions/checkout@v2 26 | 27 | - uses: actions/cache@v1 28 | with: 29 | path: ~/go/pkg/mod 30 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 31 | restore-keys: | 32 | ${{ runner.os }}-go- 33 | 34 | - name: Run tests 35 | run: make lint test 36 | 37 | build: 38 | name: Build artifacts 39 | runs-on: ubuntu-latest 40 | steps: 41 | 42 | - uses: actions/checkout@v1 43 | 44 | - name: Set up Go 1.15 45 | uses: actions/setup-go@v1 46 | with: 47 | go-version: 1.15 48 | id: go 49 | 50 | - name: Check out code into the Go module directory 51 | uses: actions/checkout@v2 52 | 53 | - uses: actions/cache@v1 54 | with: 55 | path: ~/go/pkg/mod 56 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 57 | restore-keys: | 58 | ${{ runner.os }}-go- 59 | 60 | - name: Build artifacts 61 | run: make dist 62 | -------------------------------------------------------------------------------- /pkg/cmd/uninstall.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/tinyzimmer/k3p/pkg/cluster/node" 7 | "github.com/tinyzimmer/k3p/pkg/log" 8 | ) 9 | 10 | func init() { 11 | rootCmd.AddCommand(uninstallCmd) 12 | } 13 | 14 | func completeClusters(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { 15 | log.Verbose = false 16 | clusters, err := node.ListDockerClusters() 17 | if err != nil { 18 | return nil, cobra.ShellCompDirectiveError 19 | } 20 | return clusters, cobra.ShellCompDirectiveDefault 21 | } 22 | 23 | var uninstallCmd = &cobra.Command{ 24 | Use: "uninstall", 25 | Short: "Uninstall a k3p package (currently only for docker)", 26 | Args: cobra.ExactArgs(1), 27 | ValidArgsFunction: completeClusters, 28 | RunE: func(cmd *cobra.Command, args []string) error { 29 | uninstallName := args[0] 30 | nodes, err := node.LoadDockerCluster(uninstallName) 31 | if err != nil { 32 | return err 33 | } 34 | if len(nodes) == 0 { 35 | log.Info("No running clusters found for", uninstallName) 36 | return nil 37 | } 38 | log.Info("Removing docker cluster", uninstallName) 39 | for _, dockerNode := range nodes { 40 | defer dockerNode.Close() 41 | if err := dockerNode.RemoveAll(); err != nil { 42 | return err 43 | } 44 | } 45 | log.Info("Removing docker network", uninstallName) 46 | return node.DeleteDockerNetwork(uninstallName) 47 | }, 48 | } 49 | -------------------------------------------------------------------------------- /examples/ha/whoami.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: whoami 6 | namespace: default 7 | labels: 8 | app: whoami 9 | spec: 10 | replicas: 3 11 | selector: 12 | matchLabels: 13 | app: whoami 14 | template: 15 | metadata: 16 | labels: 17 | app: whoami 18 | annotations: 19 | spec: 20 | containers: 21 | - name: whoami 22 | image: "traefik/whoami:latest" 23 | imagePullPolicy: Never 24 | ports: 25 | - name: http 26 | containerPort: 80 27 | affinity: 28 | podAntiAffinity: 29 | requiredDuringSchedulingIgnoredDuringExecution: 30 | - labelSelector: 31 | matchExpressions: 32 | - key: app 33 | operator: In 34 | values: 35 | - whoami 36 | topologyKey: "kubernetes.io/hostname" 37 | --- 38 | apiVersion: v1 39 | kind: Service 40 | metadata: 41 | name: whoami 42 | namespace: default 43 | labels: 44 | app: whoami 45 | spec: 46 | type: ClusterIP 47 | ports: 48 | - name: http 49 | port: 80 50 | targetPort: 80 51 | selector: 52 | app: whoami 53 | --- 54 | kind: Ingress 55 | apiVersion: extensions/v1beta1 56 | metadata: 57 | name: whoami 58 | namespace: default 59 | spec: 60 | rules: 61 | - host: localhost 62 | http: 63 | paths: 64 | - path: / 65 | backend: 66 | serviceName: whoami 67 | servicePort: 80 68 | -------------------------------------------------------------------------------- /doc/k3p_completion.md: -------------------------------------------------------------------------------- 1 | ## k3p completion 2 | 3 | Generate completion script 4 | 5 | ### Synopsis 6 | 7 | To load completions: 8 | 9 | Bash: 10 | 11 | $ source <(k3p completion bash) 12 | 13 | # To load completions for each session, execute once: 14 | Linux: 15 | $ k3p completion bash > /etc/bash_completion.d/k3p 16 | MacOS: 17 | $ k3p completion bash > /usr/local/etc/bash_completion.d/k3p 18 | 19 | Zsh: 20 | 21 | # If shell completion is not already enabled in your environment you will need 22 | # to enable it. You can execute the following once: 23 | 24 | $ echo "autoload -U compinit; compinit" >> ~/.zshrc 25 | 26 | # To load completions for each session, execute once: 27 | $ k3p completion zsh > "${fpath[1]}/_k3p" 28 | 29 | # You will need to start a new shell for this setup to take effect. 30 | 31 | Fish: 32 | 33 | $ k3p completion fish | source 34 | 35 | # To load completions for each session, execute once: 36 | $ k3p completion fish > ~/.config/fish/completions/k3p.fish 37 | 38 | 39 | ``` 40 | k3p completion [bash|zsh|fish|powershell] 41 | ``` 42 | 43 | ### Options 44 | 45 | ``` 46 | -h, --help help for completion 47 | ``` 48 | 49 | ### Options inherited from parent commands 50 | 51 | ``` 52 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 53 | --tmp-dir string Override the default tmp directory (default "/tmp") 54 | -v, --verbose Enable verbose logging 55 | ``` 56 | 57 | ### SEE ALSO 58 | 59 | * [k3p](k3p.md) - k3p is a k3s packaging and delivery utility 60 | 61 | -------------------------------------------------------------------------------- /doc/k3p_node.md: -------------------------------------------------------------------------------- 1 | ## k3p node 2 | 3 | Node management commands 4 | 5 | ### Options 6 | 7 | ``` 8 | -h, --help help for node 9 | -L, --leader string The IP address or DNS name of the leader of the cluster. 10 | 11 | When left unset, the machine running k3p is assumed to be the leader of the cluster. Otherwise, 12 | the provided host is remoted into, with the same connection options as for the new node in case 13 | of an add, to retrieve the installation manifest. 14 | 15 | -k, --private-key string A private key to use for SSH authentication, if not provided you will be prompted for a password (default "/home//.ssh/id_rsa") 16 | -p, --ssh-port int The port to use when connecting to the remote instance over SSH (default 22) 17 | -u, --ssh-user string The remote user to use for SSH authentication (default "") 18 | ``` 19 | 20 | ### Options inherited from parent commands 21 | 22 | ``` 23 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 24 | --tmp-dir string Override the default tmp directory (default "/tmp") 25 | -v, --verbose Enable verbose logging 26 | ``` 27 | 28 | ### SEE ALSO 29 | 30 | * [k3p](k3p.md) - k3p is a k3s packaging and delivery utility 31 | * [k3p node add](k3p_node_add.md) - Add a new node to the cluster 32 | * [k3p node remove](k3p_node_remove.md) - Remove a node from the cluster by name or IP 33 | 34 | -------------------------------------------------------------------------------- /doc/k3p_node_add.md: -------------------------------------------------------------------------------- 1 | ## k3p node add 2 | 3 | Add a new node to the cluster 4 | 5 | ``` 6 | k3p node add NODE [flags] 7 | ``` 8 | 9 | ### Options 10 | 11 | ``` 12 | -h, --help help for add 13 | -r, --node-role string Whether to join the instance as a 'server' or 'agent' (default "agent") 14 | ``` 15 | 16 | ### Options inherited from parent commands 17 | 18 | ``` 19 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 20 | -L, --leader string The IP address or DNS name of the leader of the cluster. 21 | 22 | When left unset, the machine running k3p is assumed to be the leader of the cluster. Otherwise, 23 | the provided host is remoted into, with the same connection options as for the new node in case 24 | of an add, to retrieve the installation manifest. 25 | 26 | -k, --private-key string A private key to use for SSH authentication, if not provided you will be prompted for a password (default "/home//.ssh/id_rsa") 27 | -p, --ssh-port int The port to use when connecting to the remote instance over SSH (default 22) 28 | -u, --ssh-user string The remote user to use for SSH authentication (default "") 29 | --tmp-dir string Override the default tmp directory (default "/tmp") 30 | -v, --verbose Enable verbose logging 31 | ``` 32 | 33 | ### SEE ALSO 34 | 35 | * [k3p node](k3p_node.md) - Node management commands 36 | 37 | -------------------------------------------------------------------------------- /doc/k3p_node_remove.md: -------------------------------------------------------------------------------- 1 | ## k3p node remove 2 | 3 | Remove a node from the cluster by name or IP 4 | 5 | ``` 6 | k3p node remove NODE [flags] 7 | ``` 8 | 9 | ### Options 10 | 11 | ``` 12 | -h, --help help for remove 13 | --uninstall After the node is removed from the cluster, remote in and uninstall k3s 14 | ``` 15 | 16 | ### Options inherited from parent commands 17 | 18 | ``` 19 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 20 | -L, --leader string The IP address or DNS name of the leader of the cluster. 21 | 22 | When left unset, the machine running k3p is assumed to be the leader of the cluster. Otherwise, 23 | the provided host is remoted into, with the same connection options as for the new node in case 24 | of an add, to retrieve the installation manifest. 25 | 26 | -k, --private-key string A private key to use for SSH authentication, if not provided you will be prompted for a password (default "/home//.ssh/id_rsa") 27 | -p, --ssh-port int The port to use when connecting to the remote instance over SSH (default 22) 28 | -u, --ssh-user string The remote user to use for SSH authentication (default "") 29 | --tmp-dir string Override the default tmp directory (default "/tmp") 30 | -v, --verbose Enable verbose logging 31 | ``` 32 | 33 | ### SEE ALSO 34 | 35 | * [k3p node](k3p_node.md) - Node management commands 36 | 37 | -------------------------------------------------------------------------------- /pkg/types/registry_options.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import "fmt" 4 | 5 | // BuildRegistryOptions are options for configuring an in-cluster private 6 | // container registry. 7 | type BuildRegistryOptions struct { 8 | // A name to use when generating identifiers for various resources 9 | Name string 10 | // The version of the application this registry is being built for. 11 | // Defaults to latest. 12 | AppVersion string 13 | // A list of images to bundle in the registry 14 | Images []string 15 | // Architecture to build the registry for 16 | Arch string 17 | // Pull policy to use while building the registry 18 | PullPolicy PullPolicy 19 | } 20 | 21 | // RegistryImageName returns the name to use for the image containing the registry contents. 22 | func (opts *BuildRegistryOptions) RegistryImageName() string { 23 | return fmt.Sprintf("%s-private-registry-data:%s", opts.Name, opts.AppVersion) 24 | } 25 | 26 | // RegistryTLSOptions repsent options to use when generating TLS secrets for an in-cluster 27 | // private registry. 28 | type RegistryTLSOptions struct { 29 | // A name to use when generating self-signed certificates 30 | Name string 31 | // The path to a TLS certificate to use for the private registry. If left unset a 32 | // self-signed certificate chain is generated. 33 | RegistryTLSCertFile string 34 | // The path to an unencrypted TLS private key to use for the private registry that matches 35 | // the leaf certificate provided to RegistryTLSBundle. A key is generated if not provided. 36 | RegistryTLSKeyFile string 37 | // The path to the CA bundle for the provided TLS certificate 38 | RegistryTLSCAFile string 39 | } 40 | -------------------------------------------------------------------------------- /pkg/types/artifact.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | ) 10 | 11 | // Artifact represents an object to be placed or extracted from a bundle. 12 | type Artifact struct { 13 | // The type of the artifact 14 | Type ArtifactType 15 | // The name of the artifact (this can include subdirectories) 16 | Name string 17 | // The size of the artifact 18 | Size int64 19 | // The contents of the artifact 20 | Body io.ReadCloser 21 | } 22 | 23 | // Verify will verify the contents of this artifact against the given sha256sum. 24 | // Note that this method will read the entire contents of the artifact into memory. 25 | func (a *Artifact) Verify(sha256sum string) error { 26 | var buf bytes.Buffer 27 | defer func() { a.Body = ioutil.NopCloser(&buf) }() 28 | tee := io.TeeReader(a.Body, &buf) 29 | defer a.Body.Close() // will pop off the stack first 30 | h := sha256.New() 31 | if _, err := io.Copy(h, tee); err != nil { 32 | return err 33 | } 34 | localSum := fmt.Sprintf("%x", h.Sum(nil)) 35 | if localSum != sha256sum { 36 | return fmt.Errorf("sha256 mismatch in %s %s", a.Type, a.Name) 37 | } 38 | return nil 39 | } 40 | 41 | // ApplyVariables will template this artifact's body with the given variables. 42 | func (a *Artifact) ApplyVariables(vars map[string]string) error { 43 | defer a.Body.Close() 44 | body, err := ioutil.ReadAll(a.Body) 45 | if err != nil { 46 | return err 47 | } 48 | body, err = render(body, vars) 49 | if err != nil { 50 | return err 51 | } 52 | a.Body = ioutil.NopCloser(bytes.NewReader(body)) 53 | a.Size = int64(len(body)) 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /pkg/types/cluster.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // NodeConnectOptions are options for configuring a connection to a remote node. 4 | type NodeConnectOptions struct { 5 | // The user to attempt to SSH into the remote node as. 6 | SSHUser string 7 | // A password to use for SSH authentication. 8 | SSHPassword string 9 | // The path to the key to use for SSH authentication. 10 | SSHKeyFile string 11 | // The port to use for the SSH connection 12 | SSHPort int 13 | // The address of the new node. 14 | Address string 15 | } 16 | 17 | // AddNodeOptions represents options passed to the AddNode operation. 18 | type AddNodeOptions struct { 19 | // Options for remote connections 20 | *NodeConnectOptions 21 | // The role to assign the new node. 22 | NodeRole K3sRole 23 | } 24 | 25 | // RemoveNodeOptions are options passed to a RemoveNode operation (not implemented). 26 | type RemoveNodeOptions struct { 27 | // Options for remote connections 28 | *NodeConnectOptions 29 | // Attempt to remote into the system and uninstall k3s 30 | Uninstall bool 31 | // The name of the node to remove 32 | Name string 33 | // The IP address of the node to remove 34 | IPAddress string 35 | } 36 | 37 | // ClusterManager is an interface for managing the nodes in a k3s cluster. 38 | type ClusterManager interface { 39 | // AddNode should add a new node to the k3s cluster. 40 | AddNode(Node, *AddNodeOptions) error 41 | // RemoveNode should drain and remove the given node from the k3s cluster. 42 | // If NodeConnectOptions are not nil and Uninstall is true, then k3s and 43 | // all of its assets should be completely removed from the system. (not implemented) 44 | RemoveNode(*RemoveNodeOptions) error 45 | } 46 | -------------------------------------------------------------------------------- /pkg/cluster/node/mock.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/tinyzimmer/k3p/pkg/types" 12 | ) 13 | 14 | // Mock returns a mock node rooting all files from a temp directory. 15 | func Mock() types.Node { 16 | tmpDir, err := ioutil.TempDir("", "") 17 | if err != nil { 18 | panic(err) // Mock would not be used in normal execution so this is fine 19 | } 20 | return &mockNode{root: tmpDir} 21 | } 22 | 23 | type mockNode struct{ root string } 24 | 25 | func (m *mockNode) rootedDir(f string) string { 26 | return path.Join(m.root, strings.TrimPrefix(f, "/")) 27 | } 28 | 29 | func (m *mockNode) GetType() types.NodeType { return types.NodeLocal } 30 | 31 | func (m *mockNode) Close() error { return os.RemoveAll(m.root) } 32 | 33 | func (m *mockNode) Execute(opts *types.ExecuteOptions) error { return nil } 34 | 35 | func (m *mockNode) GetFile(f string) (io.ReadCloser, error) { 36 | return os.Open(m.rootedDir(f)) 37 | } 38 | 39 | func (m *mockNode) WriteFile(rdr io.ReadCloser, dest, mode string, size int64) error { 40 | defer rdr.Close() 41 | if err := m.MkdirAll(path.Dir(dest)); err != nil { 42 | return err 43 | } 44 | u, err := strconv.ParseUint(mode, 0, 16) 45 | if err != nil { 46 | return err 47 | } 48 | f, err := os.OpenFile(m.rootedDir(dest), os.O_RDWR|os.O_CREATE, os.FileMode(u)) 49 | if err != nil { 50 | return err 51 | } 52 | defer f.Close() 53 | _, err = io.Copy(f, rdr) 54 | return err 55 | } 56 | 57 | func (m *mockNode) MkdirAll(path string) error { return os.MkdirAll(m.rootedDir(path), 0755) } 58 | 59 | func (m *mockNode) GetK3sAddress() (string, error) { return "", nil } 60 | -------------------------------------------------------------------------------- /pkg/cmd/completion.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | func init() { 10 | rootCmd.AddCommand(completionCmd) 11 | } 12 | 13 | var completionCmd = &cobra.Command{ 14 | Use: "completion [bash|zsh|fish|powershell]", 15 | Short: "Generate completion script", 16 | Long: `To load completions: 17 | 18 | Bash: 19 | 20 | $ source <(k3p completion bash) 21 | 22 | # To load completions for each session, execute once: 23 | Linux: 24 | $ k3p completion bash > /etc/bash_completion.d/k3p 25 | MacOS: 26 | $ k3p completion bash > /usr/local/etc/bash_completion.d/k3p 27 | 28 | Zsh: 29 | 30 | # If shell completion is not already enabled in your environment you will need 31 | # to enable it. You can execute the following once: 32 | 33 | $ echo "autoload -U compinit; compinit" >> ~/.zshrc 34 | 35 | # To load completions for each session, execute once: 36 | $ k3p completion zsh > "${fpath[1]}/_k3p" 37 | 38 | # You will need to start a new shell for this setup to take effect. 39 | 40 | Fish: 41 | 42 | $ k3p completion fish | source 43 | 44 | # To load completions for each session, execute once: 45 | $ k3p completion fish > ~/.config/fish/completions/k3p.fish 46 | `, 47 | DisableFlagsInUseLine: true, 48 | ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, 49 | Args: cobra.ExactValidArgs(1), 50 | Run: func(cmd *cobra.Command, args []string) { 51 | switch args[0] { 52 | case "bash": 53 | cmd.Root().GenBashCompletion(os.Stdout) 54 | case "zsh": 55 | cmd.Root().GenZshCompletion(os.Stdout) 56 | case "fish": 57 | cmd.Root().GenFishCompletion(os.Stdout, true) 58 | case "powershell": 59 | cmd.Root().GenPowerShellCompletion(os.Stdout) 60 | } 61 | }, 62 | } 63 | -------------------------------------------------------------------------------- /pkg/types/manifest.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // Manifest contains the listings of all the files in the package. 4 | type Manifest struct { 5 | // Binaries inside the package 6 | Bins []string `json:"bins,omitempty"` 7 | // Scripts inside the package 8 | Scripts []string `json:"scripts,omitempty"` 9 | // Images inside the package 10 | Images []string `json:"images,omitempty"` 11 | // Kubernetes manifests inside the package 12 | K8sManifests []string `json:"k8sManifests,omitempty"` 13 | // Static assets 14 | Static []string `json:"static,omitempty"` 15 | // Etc assets 16 | Etc []string `json:"etc,omitempty"` 17 | // The End User License Agreement for the package, or an empty string if there is none 18 | EULA string `json:"eula,omitempty"` 19 | } 20 | 21 | // DeepCopy returns a copy of this Manifest. 22 | func (m *Manifest) DeepCopy() *Manifest { 23 | out := &Manifest{ 24 | Bins: make([]string, len(m.Bins)), 25 | Scripts: make([]string, len(m.Scripts)), 26 | Images: make([]string, len(m.Images)), 27 | K8sManifests: make([]string, len(m.K8sManifests)), 28 | Static: make([]string, len(m.Static)), 29 | Etc: make([]string, len(m.Etc)), 30 | EULA: m.EULA, 31 | } 32 | copy(out.Bins, m.Bins) 33 | copy(out.Scripts, m.Scripts) 34 | copy(out.Images, m.Images) 35 | copy(out.K8sManifests, m.K8sManifests) 36 | copy(out.Static, m.Static) 37 | copy(out.Etc, m.Etc) 38 | return out 39 | } 40 | 41 | // HasEULA returns true if the manifest contains an end user license agreement. 42 | func (m *Manifest) HasEULA() bool { return m.EULA != "" } 43 | 44 | // NewEmptyManifest initializes a manifest with empty slices. 45 | func NewEmptyManifest() *Manifest { 46 | return &Manifest{ 47 | Bins: make([]string, 0), 48 | Scripts: make([]string, 0), 49 | Images: make([]string, 0), 50 | K8sManifests: make([]string, 0), 51 | Static: make([]string, 0), 52 | Etc: make([]string, 0), 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pkg/types/package.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // Package is an interface to be implemented for use by a package bundler/extracter. 8 | // Different versions of how packages are built can implement this interface. 9 | type Package interface { 10 | // Put should store the provided artifact inside the archive. The interface is responsible 11 | // for appending the details of the artifact to the metadata. 12 | Put(*Artifact) error 13 | // PutMeta should merge the provided meta with any tracked internally by the interface. 14 | PutMeta(meta *PackageMeta) error 15 | // Read should populate the given artifact with the contents inside the archive. 16 | Get(*Artifact) error 17 | // GetMeta should return the metadata associated with the package. This will contain information 18 | // on the full contents of the package. 19 | GetMeta() *PackageMeta 20 | // Archive should produce an Archive interface that can be used to read from the final package stream. 21 | // This method should ensure any metadata and finalize the archive. Any changes made to the package after 22 | // Archive is called will require another call to receive the latest changes. 23 | Archive() (Archive, error) 24 | // Close should perform any necessary cleanup on both this interface, and archives created from it. 25 | Close() error 26 | } 27 | 28 | // Archive is an interface to be implemented by packagers/extracers. It contains the final contents 29 | // of the archive and methods for interacting with it. 30 | type Archive interface { 31 | // Reader should return a simple io.ReadCloser for the archive. 32 | Reader() io.ReadCloser 33 | // WriteTo should dump the contents of the archive to the given file. 34 | WriteTo(path string) error 35 | // CompressTo should compress the contents of the archive to the given zst file. 36 | CompressTo(path string) error 37 | // CompressReader should return an io.ReadCloser who's contents are compressed 38 | // with zstandard. The value returned by Size() will not accurately reflect the 39 | // contents of this Reader. 40 | CompressReader() (io.ReadCloser, error) 41 | // Size should return the size of the archive. 42 | Size() int64 43 | } 44 | -------------------------------------------------------------------------------- /pkg/cluster/node/local.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "os/exec" 7 | "path" 8 | "strconv" 9 | 10 | "github.com/tinyzimmer/k3p/pkg/log" 11 | "github.com/tinyzimmer/k3p/pkg/types" 12 | "github.com/tinyzimmer/k3p/pkg/util" 13 | ) 14 | 15 | // Local returns a new Node pointing at the local system. 16 | func Local() types.Node { 17 | return &localNode{} 18 | } 19 | 20 | type localNode struct{} 21 | 22 | func (l *localNode) GetType() types.NodeType { return types.NodeLocal } 23 | 24 | func (l *localNode) MkdirAll(dir string) error { 25 | log.Debugf("Ensuring local system directory %q with mode 0755\n", dir) 26 | return os.MkdirAll(dir, 0755) 27 | } 28 | 29 | func (l *localNode) Close() error { return nil } 30 | 31 | func (l *localNode) GetFile(f string) (io.ReadCloser, error) { return os.Open(f) } 32 | 33 | // size is ignored for local nodes 34 | func (l *localNode) WriteFile(rdr io.ReadCloser, dest string, mode string, size int64) error { 35 | defer rdr.Close() 36 | if err := l.MkdirAll(path.Dir(dest)); err != nil { 37 | return err 38 | } 39 | log.Debugf("Writing file to local system at %q with mode %q\n", dest, mode) 40 | u, err := strconv.ParseUint(mode, 0, 16) 41 | if err != nil { 42 | return err 43 | } 44 | f, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, os.FileMode(u)) 45 | if err != nil { 46 | return err 47 | } 48 | defer f.Close() 49 | _, err = io.Copy(f, rdr) 50 | return err 51 | } 52 | 53 | func (l *localNode) Execute(opts *types.ExecuteOptions) error { 54 | cmd := buildCmdFromExecOpts(opts) 55 | log.Debug("Executing command on local system:", redactSecrets(cmd, opts.Secrets)) 56 | c := exec.Command("/bin/sh", "-c", cmd) 57 | outPipe, err := c.StdoutPipe() 58 | if err != nil { 59 | return err 60 | } 61 | errPipe, err := c.StderrPipe() 62 | if err != nil { 63 | return err 64 | } 65 | go log.LevelReader(log.LevelInfo, outPipe) 66 | go log.LevelReader(log.LevelDebug, errPipe) 67 | return c.Run() 68 | } 69 | 70 | func (l *localNode) GetK3sAddress() (string, error) { 71 | addr, err := util.GetExternalAddressForProcess("k3s-server") 72 | if err != nil { 73 | return "", err 74 | } 75 | return addr.String(), nil 76 | } 77 | -------------------------------------------------------------------------------- /pkg/cmd/token.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "strings" 9 | 10 | "github.com/spf13/cobra" 11 | "github.com/tinyzimmer/k3p/pkg/types" 12 | "github.com/tinyzimmer/k3p/pkg/util" 13 | ) 14 | 15 | var generatedTokenLength int 16 | 17 | func init() { 18 | tokenGenerateCmd.Flags().IntVarP(&generatedTokenLength, "length", "l", 128, "The length of the token to generate") 19 | 20 | tokenCmd.AddCommand(tokenGetCmd) 21 | tokenCmd.AddCommand(tokenGenerateCmd) 22 | rootCmd.AddCommand(tokenCmd) 23 | } 24 | 25 | var tokenCmd = &cobra.Command{ 26 | Use: "token", 27 | Short: "Token retrieval and generation commands", 28 | } 29 | 30 | var tokenGetCmd = &cobra.Command{ 31 | Use: "get TOKEN_TYPE", 32 | Short: "Retrieve a k3s token", 33 | Long: ` 34 | Retrieves the token for joining either a new "agent" or "server" to the cluster. 35 | 36 | The "agent" token can be retrieved from any of the server instances, while the "server" token 37 | can only be retrieved on the server where "k3p install" was run with "--init-ha". 38 | `, 39 | Args: cobra.ExactValidArgs(1), 40 | ValidArgs: []string{"agent", "server"}, 41 | RunE: func(cmd *cobra.Command, args []string) error { 42 | switch args[0] { 43 | case "agent": 44 | token, err := ioutil.ReadFile(types.AgentTokenFile) 45 | if err != nil { 46 | if os.IsNotExist(err) { 47 | return errors.New("The K3s server does not appear to be installed to the system") 48 | } 49 | return err 50 | } 51 | fmt.Println(strings.TrimSpace(string(token))) 52 | case "server": 53 | token, err := ioutil.ReadFile(types.ServerTokenFile) 54 | if err != nil { 55 | if os.IsNotExist(err) { 56 | return errors.New("This system does not appear to have been initialized with --init-ha") 57 | } 58 | return err 59 | } 60 | fmt.Println(strings.TrimSpace(string(token))) 61 | } 62 | return nil 63 | }, 64 | } 65 | 66 | var tokenGenerateCmd = &cobra.Command{ 67 | Use: "generate", 68 | Short: "Generates a token that can be used for initializing HA installations", 69 | Run: func(cmd *cobra.Command, args []string) { 70 | fmt.Println(util.GenerateToken(generatedTokenLength)) 71 | }, 72 | } 73 | -------------------------------------------------------------------------------- /examples/helm-charts/README.md: -------------------------------------------------------------------------------- 1 | # Helm chart example 2 | 3 | This directory contains a config that can be used for a build with helm charts. 4 | First you must pull the helm charts down in to this directory (you can also use chart directories as well): 5 | 6 | ```sh 7 | # Add the helm chart repositories 8 | $ helm repo add tinyzimmer https://tinyzimmer.github.io/kvdi/deploy/charts 9 | $ helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 10 | $ helm repo update 11 | 12 | # Download the packaged charts 13 | $ helm fetch prometheus-community/kube-prometheus-stack 14 | $ helm fetch tinyzimmer/kvdi 15 | ``` 16 | 17 | Then build a package in this directory using the provided config: 18 | 19 | ```sh 20 | # For the sake of producing a smaller artifact, we'll use the --exclude-images flag 21 | $ k3p build --exclude-images --name kvdi 22 | 2020/12/11 20:32:08 [INFO] Building package "kvdi" 23 | 2020/12/11 20:32:08 [INFO] Detecting latest k3s version for channel stable 24 | 2020/12/11 20:32:09 [INFO] Latest k3s version is v1.19.4+k3s1 25 | 2020/12/11 20:32:09 [INFO] Packaging distribution for version "v1.19.4+k3s1" using "amd64" architecture 26 | 2020/12/11 20:32:09 [INFO] Downloading core k3s components 27 | 2020/12/11 20:32:09 [INFO] Fetching checksums... 28 | 2020/12/11 20:32:09 [INFO] Fetching k3s install script... 29 | 2020/12/11 20:32:09 [INFO] Fetching k3s binary... 30 | 2020/12/11 20:32:09 [INFO] Skipping bundling k3s airgap images with the package 31 | 2020/12/11 20:32:09 [INFO] Validating checksums... 32 | 2020/12/11 20:32:09 [INFO] Searching "/home/tinyzimmer/devel/k3p/examples/kvdi" for kubernetes manifests to include in the archive 33 | 2020/12/11 20:32:09 [INFO] Detected helm chart at /home/tinyzimmer/devel/k3p/examples/kvdi/kube-prometheus-stack-12.8.0.tgz 34 | 2020/12/11 20:32:09 [INFO] Detected helm chart at /home/tinyzimmer/devel/k3p/examples/kvdi/kvdi-v0.1.1.tgz 35 | 2020/12/11 20:32:09 [INFO] Skipping bundling container images with the package 36 | 2020/12/11 20:32:09 [INFO] Writing package metadata 37 | 2020/12/11 20:32:09 [INFO] Archiving version "latest" of "kvdi" to "/home/tinyzimmer/devel/k3p/examples/kvdi/package.tar" 38 | ``` 39 | -------------------------------------------------------------------------------- /pkg/types/builder.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // Builder is an interface for building application bundles to be distributed to systems. 4 | type Builder interface { 5 | Build(*BuildOptions) error 6 | } 7 | 8 | // PullPolicy represents the pull policy to use when bundling images 9 | // TODO: This should probably be pulled from corev1. 10 | type PullPolicy string 11 | 12 | // Valid pull policies 13 | const ( 14 | PullPolicyAlways PullPolicy = "always" 15 | PullPolicyNever PullPolicy = "never" 16 | PullPolicyIfNotPresent PullPolicy = "ifnotpresent" 17 | ) 18 | 19 | // BuildOptions is a struct containing options to pass to the build operation. 20 | type BuildOptions struct { 21 | // The version of the package being built 22 | BuildVersion string 23 | // The name of the package, if not provided one is generated using docker's name generator 24 | Name string 25 | // The version of K3s to bundle with the package, overrides K3sChannel 26 | K3sVersion string 27 | // The release channel to retrieve the latest K3s version from 28 | K3sChannel string 29 | // The CPU architecture to target the package for 30 | Arch string 31 | // An optional EULA to provide with the package 32 | EULAFile string 33 | // An optional config file providing variables to be used at installation 34 | ConfigFile string 35 | // A path to an optional file of newline delimited container images to include in the package 36 | ImageFile string 37 | // A list of images to include in the package 38 | Images []string 39 | // The directory to scan for kubernetes manifests and helm charts 40 | ManifestDirs []string 41 | // A list of directories to exclude while searching for manifests 42 | Excludes []string 43 | // Don't bundle docker images with the archive 44 | ExcludeImages bool 45 | // When true, instead of creating a tarball of images that is installed to every agent, a private 46 | // registry is built and the package is configured to launch and use it at installation. 47 | CreateRegistry bool 48 | // The pull policy to use 49 | PullPolicy PullPolicy 50 | // The path to write the final archive to 51 | Output string 52 | // Whether to apply zst compression to the final archive 53 | Compress bool 54 | // Whether to write the outputs to a self-installing run file 55 | RunFile bool 56 | } 57 | -------------------------------------------------------------------------------- /pkg/cluster/kubernetes/client.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/tinyzimmer/k3p/pkg/types" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/client-go/kubernetes" 12 | "k8s.io/client-go/tools/clientcmd" 13 | ) 14 | 15 | // Client is a kubernetes client abstraction for k3s management operations. 16 | type Client interface { 17 | GetNodeByIP(ip string) (*corev1.Node, error) 18 | GetIPByNodeName(name string) (string, error) 19 | ListNodes() ([]corev1.Node, error) 20 | RemoveNode(name string) error 21 | } 22 | 23 | // New returns a new Client for the k3s cluster using the given kubeconfig bytes 24 | func New(cfg []byte) (Client, error) { 25 | config, err := clientcmd.RESTConfigFromKubeConfig(cfg) 26 | if err != nil { 27 | return nil, err 28 | } 29 | clientset, err := kubernetes.NewForConfig(config) 30 | if err != nil { 31 | return nil, err 32 | } 33 | return &client{clientset}, nil 34 | } 35 | 36 | type client struct { 37 | clientset *kubernetes.Clientset 38 | } 39 | 40 | func (c *client) GetIPByNodeName(name string) (string, error) { 41 | node, err := c.clientset. 42 | CoreV1(). 43 | Nodes(). 44 | Get(context.TODO(), name, metav1.GetOptions{}) 45 | if err != nil { 46 | return "", err 47 | } 48 | labels := node.GetLabels() 49 | if ip, ok := labels[types.K3sInternalIPLabel]; ok { 50 | return ip, nil 51 | } 52 | return "", fmt.Errorf("Node %q is missing k3s internal IP label", name) 53 | } 54 | 55 | func (c *client) GetNodeByIP(ip string) (*corev1.Node, error) { 56 | nodeList, err := c.clientset. 57 | CoreV1(). 58 | Nodes(). 59 | List(context.TODO(), metav1.ListOptions{ 60 | LabelSelector: fmt.Sprintf("%s=%s", types.K3sInternalIPLabel, ip), 61 | }) 62 | if err != nil { 63 | return nil, err 64 | } 65 | if len(nodeList.Items) == 0 { 66 | return nil, fmt.Errorf("No node with the IP %q found in the cluster", ip) 67 | } 68 | return &nodeList.Items[0], nil 69 | } 70 | 71 | func (c *client) ListNodes() ([]corev1.Node, error) { 72 | nodeList, err := c.clientset. 73 | CoreV1(). 74 | Nodes(). 75 | List(context.TODO(), metav1.ListOptions{}) 76 | if err != nil { 77 | return nil, err 78 | } 79 | return nodeList.Items, nil 80 | } 81 | 82 | func (c *client) RemoveNode(name string) error { 83 | return c.clientset. 84 | CoreV1(). 85 | Nodes(). 86 | Delete(context.TODO(), name, metav1.DeleteOptions{}) 87 | } 88 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DIST ?= $(CURDIR)/dist 2 | BIN ?= $(DIST)/k3p 3 | GOPATH ?= $(shell go env GOPATH) 4 | GOBIN ?= $(GOPATH)/bin 5 | 6 | GOOS ?= linux 7 | CGO_ENABLED ?= 0 8 | 9 | GOLANGCI_VERSION ?= v1.33.0 10 | GOLANGCI_LINT ?= $(GOBIN)/golangci-lint 11 | GINKGO ?= $(GOBIN)/ginkgo 12 | GOX ?= $(GOBIN)/gox 13 | UPX ?= $(shell which upx 2> /dev/null) 14 | 15 | VERSION ?= $(shell git describe --tags) 16 | COMMIT ?= $(shell git rev-parse HEAD) 17 | ZST_DICT ?= $(CURDIR)/hack/zstDictionary 18 | 19 | LDFLAGS ?= "-X github.com/tinyzimmer/k3p/pkg/build/package/v1.ZstDictionaryB64=`cat '$(ZST_DICT)' | base64 --wrap=0` \ 20 | -X github.com/tinyzimmer/k3p/pkg/version.K3pVersion=$(VERSION) \ 21 | -X github.com/tinyzimmer/k3p/pkg/version.K3pCommit=$(COMMIT) -s -w" 22 | 23 | COMPRESSION ?= 5 24 | 25 | build: $(BIN) 26 | 27 | $(BIN): 28 | cd cmd/k3p && \ 29 | CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) \ 30 | go build -o $(BIN) \ 31 | -ldflags $(LDFLAGS) 32 | ifneq ($(UPX),) 33 | $(UPX) -$(COMPRESSION) $(BIN) 34 | endif 35 | 36 | IMG ?= ghcr.io/tinyzimmer/k3p:$(shell git describe --tags) 37 | docker: 38 | docker build . -t $(IMG) 39 | 40 | $(GOX): 41 | GO111MODULE=off go get github.com/mitchellh/gox 42 | 43 | .PHONY: dist 44 | COMPILE_TARGETS ?= "darwin/amd64 linux/amd64 linux/arm linux/arm64 windows/amd64" 45 | COMPILE_OUTPUT ?= "$(DIST)/{{.Dir}}_{{.OS}}_{{.Arch}}" 46 | dist: $(GOX) 47 | cd cmd/k3p && \ 48 | CGO_ENABLED=$(CGO_ENABLED) $(GOX) -osarch $(COMPILE_TARGETS) --output $(COMPILE_OUTPUT) -ldflags=$(LDFLAGS) 49 | ifneq ($(UPX),) 50 | $(UPX) -$(COMPRESSION) $(DIST)/* 51 | endif 52 | 53 | 54 | install: $(BIN) 55 | mkdir -p $(GOBIN) 56 | cp $(BIN) $(GOBIN)/k3p 57 | 58 | docs: 59 | go run hack/docgen.go 60 | 61 | clean: 62 | find . -name '*.coverprofile' -exec rm {} \; 63 | find . -name '*.tgz' -exec rm {} \; 64 | find . -name '*.tar' -exec rm {} \; 65 | find . -name '*.run' -exec rm {} \; 66 | rm -rf $(DIST) tls/ 67 | 68 | $(GOLANGCI_LINT): 69 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOBIN) $(GOLANGCI_VERSION) 70 | 71 | lint: $(GOLANGCI_LINT) 72 | $(GOLANGCI_LINT) run -v --timeout 300s 73 | 74 | $(GINKGO): 75 | GO111MODULE=off go get github.com/onsi/ginkgo/ginkgo 76 | 77 | TEST_PKG ?= ./... 78 | TEST_FLAGS ?= 79 | test: $(GINKGO) 80 | $(GINKGO) \ 81 | -cover -coverprofile=k3p.coverprofile -outputdir=. -coverpkg=$(TEST_PKG) \ 82 | $(TEST_FLAGS) $(TEST_PKG) 83 | go tool cover -func k3p.coverprofile 84 | 85 | tls: 86 | bash hack/gen-cert-chain.sh 87 | cat tls/intermediate-1.crt tls/ca.crt > tls/ca-bundle.crt 88 | 89 | tls-args: 90 | @echo -n '--registry-tls-cert="$(CURDIR)/tls/leaf.crt" --registry-tls-key="$(CURDIR)/tls/leaf.key" --registry-tls-ca="$(CURDIR)/tls/ca-bundle.crt"' -------------------------------------------------------------------------------- /doc/k3p_build.md: -------------------------------------------------------------------------------- 1 | ## k3p build 2 | 3 | Build a k3s distribution package 4 | 5 | ``` 6 | k3p build [flags] 7 | ``` 8 | 9 | ### Options 10 | 11 | ``` 12 | -a, --arch string The architecture to package the distribution for. Only (amd64, arm, and arm64 are supported) (default "amd64") 13 | -C, --channel string The release channel to retrieve the version of k3s from (default "stable") 14 | --compress Whether to apply zst encryption to the package, it will usually require the same k3p release to decompress. 15 | -c, --config string An optional file providing variables and other configurations to be used at installation, if a k3p.yaml in the current directory exists it will be used automatically 16 | -E, --eula string A file containing an End User License Agreement to display to the user upon installing the package 17 | -e, --exclude strings Directories to exclude when reading the manifest directory 18 | --exclude-images Don't include container images with the final archive 19 | -h, --help help for build 20 | -I, --image-file string A file containing a list of extra images to bundle with the archive 21 | -i, --images strings A comma separated list of images to include with the archive 22 | --k3s-version string A specific k3s version to bundle with the package, overrides --channel (default "latest") 23 | -m, --manifests stringArray Directories to scan for kubernetes manifests and charts, defaults to the current directory, can be specified multiple times (default [/home//devel/k3p]) 24 | -n, --name string The name to give the package, if not provided one will be generated 25 | -N, --no-cache Disable the use of the local cache when downloading assets 26 | -o, --output string The file to save the distribution package to (default "/home//devel/k3p/package.tar") 27 | --pull-policy string The pull policy to use when bundling container images (valid options always,never,ifnotpresent [case-insensitive]) (default "always") 28 | --run-file Whether to bundle the final archive into a self-installing run file 29 | --use-registry Bundle container images into a private registry instead of just raw tar balls 30 | -V, --version string The version to tag the package (default "latest") 31 | ``` 32 | 33 | ### Options inherited from parent commands 34 | 35 | ``` 36 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 37 | --tmp-dir string Override the default tmp directory (default "/tmp") 38 | -v, --verbose Enable verbose logging 39 | ``` 40 | 41 | ### SEE ALSO 42 | 43 | * [k3p](k3p.md) - k3p is a k3s packaging and delivery utility 44 | 45 | -------------------------------------------------------------------------------- /pkg/types/node.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "io" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | // NodeType represents a type of node 10 | type NodeType string 11 | 12 | const ( 13 | // NodeLocal represents the local system 14 | NodeLocal NodeType = "local" 15 | // NodeRemote represents a remote node over SSH 16 | NodeRemote NodeType = "remote" 17 | // NodeDocker represents a docker container node 18 | NodeDocker NodeType = "docker" 19 | ) 20 | 21 | // Node is an interface for preparing and managing a system that will run K3s. 22 | type Node interface { 23 | // GetType should be implemented by every node and return one of the types above 24 | GetType() NodeType 25 | // MkdirAll should ensure the given directory on the node 26 | MkdirAll(dir string) error 27 | // GetFile should retrieve the given file on the node 28 | GetFile(path string) (io.ReadCloser, error) 29 | // WriteFile should write the contents of the given reader to destination on the node, 30 | // and set its mode and size accordingly. 31 | WriteFile(rdr io.ReadCloser, destination string, mode string, size int64) error 32 | // Execute should execute a command on the node. This function should probably be renamed/repurposed 33 | // to StartK3s or something as that is all it is used for, and will make more sense in the 34 | // context of docker. 35 | Execute(*ExecuteOptions) error 36 | // GetK3sAddress should return the address where the k3s server is listening for connections 37 | // on this node. 38 | GetK3sAddress() (string, error) 39 | // Close should close any open connections to the node and perform any necessary cleanup. 40 | Close() error 41 | } 42 | 43 | // ExecuteOptions represent options to an execute command on a node. 44 | type ExecuteOptions struct { 45 | // Environment variables to set for the process 46 | Env map[string]string 47 | // The command to run 48 | Command string 49 | // Secret strings to filter from any logging output 50 | Secrets []string 51 | } 52 | 53 | // GetAPIPort returns the API port configured for these ExecuteOptions. This is a bit of a hack 54 | // for the fact that the user is allowed to pass in raw k3s exec strings. A lot of this needs to 55 | // be refactored. 56 | // What made this necessary is now taken care of, it may be safe to reevaluate this now. 57 | func (e *ExecuteOptions) GetAPIPort() int { 58 | for k, v := range e.Env { 59 | if k == "INSTALL_K3S_EXEC" { 60 | fields := strings.Fields(v) 61 | for idx, arg := range fields { 62 | if strings.HasPrefix(arg, "--https-listen-port=") { 63 | if port, err := strconv.Atoi(strings.TrimPrefix(arg, "--https-listen-port=")); err == nil { 64 | return port 65 | } 66 | } 67 | if arg == "--https-listen-port" { 68 | if port, err := strconv.Atoi(fields[idx+1]); err == nil { 69 | return port 70 | } 71 | } 72 | } 73 | } 74 | } 75 | return 6443 76 | } 77 | -------------------------------------------------------------------------------- /pkg/parser/base_parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "strings" 7 | 8 | "github.com/tinyzimmer/k3p/pkg/types" 9 | "k8s.io/apimachinery/pkg/runtime" 10 | "k8s.io/apimachinery/pkg/runtime/serializer" 11 | corescheme "k8s.io/client-go/kubernetes/scheme" 12 | ) 13 | 14 | // BaseManifestParser represents the base elements for a parser interface. It contains 15 | // convenience methods for common directory and file operations. The original intention 16 | // of this here was to be used as a base for different processors (e.g. raw, helm, kustomize, jsonnet, etc.), 17 | // however in working towards a POC it made sense to keep things simple and combine raw and helm into a single 18 | // interface. 19 | type BaseManifestParser struct { 20 | ParseDir string 21 | ExcludeDirs []string 22 | PackageConfig *types.PackageConfig 23 | Deserializer runtime.Decoder 24 | } 25 | 26 | // NewBaseManifestParser returns a new base parser with the given arguments. 27 | func NewBaseManifestParser(parseDir string, excludeDirs []string, cfg *types.PackageConfig) *BaseManifestParser { 28 | // create a new scheme 29 | sch := runtime.NewScheme() 30 | 31 | // currently only supports core APIs, could consider some way of dynamically adding CRD support 32 | // full list: https://github.com/kubernetes/client-go/blob/master/kubernetes/scheme/register.go 33 | _ = corescheme.AddToScheme(sch) 34 | 35 | return &BaseManifestParser{ 36 | ParseDir: parseDir, 37 | ExcludeDirs: excludeDirs, 38 | PackageConfig: cfg, 39 | Deserializer: serializer.NewCodecFactory(sch).UniversalDeserializer(), 40 | } 41 | } 42 | 43 | // GetParseDir returns the directory to be parsed for container images. 44 | func (b *BaseManifestParser) GetParseDir() string { return b.ParseDir } 45 | 46 | // GetHelmValues returns the raw, untemplated helm values for a given chart. Or an error 47 | // if none can be found. 48 | func (b *BaseManifestParser) GetHelmValues(chartName string) ([]byte, error) { 49 | return b.PackageConfig.RawHelmValuesForChart(chartName) 50 | } 51 | 52 | // StripParseDir is a convenience method for stripping the parse directory from the beginning 53 | // of a path. 54 | func (b *BaseManifestParser) StripParseDir(s string) string { 55 | return strings.Replace(s, b.ParseDir+"/", "", 1) 56 | } 57 | 58 | // IsExcluded returns true if the given directory should be excluded from parsing. 59 | func (b *BaseManifestParser) IsExcluded(dirName string) bool { 60 | for _, ex := range b.ExcludeDirs { 61 | if strings.TrimSuffix(ex, string(os.PathSeparator)) == strings.TrimSuffix(path.Base(dirName), string(os.PathSeparator)) { 62 | return true 63 | } 64 | } 65 | return false 66 | } 67 | 68 | // Decode will decode the given bytes into a kubernetes runtime object. 69 | func (b *BaseManifestParser) Decode(data []byte) (runtime.Object, error) { 70 | obj, _, err := b.Deserializer.Decode(data, nil, nil) 71 | return obj, err 72 | } 73 | -------------------------------------------------------------------------------- /pkg/build/package/v1/archive.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "encoding/base64" 5 | "io" 6 | "os" 7 | 8 | "github.com/klauspost/compress/zstd" 9 | "github.com/tinyzimmer/k3p/pkg/log" 10 | ) 11 | 12 | // ZstDictionaryB64 is populated at compilation and contains a pre-trained dictionary 13 | // for compressing k3s images. 14 | var ZstDictionaryB64 string 15 | 16 | func getZstDict() ([]byte, error) { 17 | return base64.StdEncoding.DecodeString(ZstDictionaryB64) 18 | } 19 | 20 | type archive struct { 21 | stat os.FileInfo 22 | f *os.File 23 | } 24 | 25 | // Reader should return a simple io.ReadCloser for the archive. 26 | func (a *archive) Reader() io.ReadCloser { return a.f } 27 | 28 | // WriteTo should dump the contents of the archive to the given file. 29 | func (a *archive) WriteTo(path string) error { 30 | out, err := os.Create(path) 31 | if err != nil { 32 | return err 33 | } 34 | defer out.Close() 35 | _, err = io.Copy(out, a.f) 36 | return err 37 | } 38 | 39 | // Size should return the size of the archive. 40 | func (a *archive) Size() int64 { return a.stat.Size() } 41 | 42 | // CompressTo will compress the contents of the archiver to the given file. 43 | // Compression is done using zstandard and a dictionary pre-trained on k3s 44 | // docker images. 45 | func (a *archive) CompressTo(path string) error { 46 | out, err := os.Create(path) 47 | if err != nil { 48 | return err 49 | } 50 | defer out.Close() 51 | crdr, err := a.CompressReader() 52 | if err != nil { 53 | return err 54 | } 55 | defer crdr.Close() 56 | _, err = io.Copy(out, crdr) 57 | return err 58 | } 59 | 60 | // CompressReader should return an io.ReadCloser who's contents are compressed 61 | // with zstandard. 62 | func (a *archive) CompressReader() (io.ReadCloser, error) { 63 | dictBytes, err := getZstDict() 64 | if err != nil { 65 | return nil, err 66 | } 67 | r, w := io.Pipe() 68 | enc, err := zstd.NewWriter(w, zstd.WithEncoderDict(dictBytes)) 69 | if err != nil { 70 | return nil, err 71 | } 72 | go func() { 73 | defer w.Close() 74 | defer enc.Close() 75 | if _, err := io.Copy(enc, a.f); err != nil { 76 | log.Error(err) 77 | } 78 | }() 79 | return r, nil 80 | } 81 | 82 | type zstReadCloser struct{ rdr *zstd.Decoder } 83 | 84 | func (z *zstReadCloser) Read(p []byte) (int, error) { return z.rdr.Read(p) } 85 | 86 | func (z *zstReadCloser) Close() error { 87 | z.rdr.Close() 88 | return nil 89 | } 90 | 91 | // Decompress will return a reader that can be used to access the decompressed contents 92 | // of a zst archive. 93 | func Decompress(rdr io.Reader) (io.ReadCloser, error) { 94 | dictBytes, err := getZstDict() 95 | if err != nil { 96 | return nil, err 97 | } 98 | out, err := zstd.NewReader(rdr, zstd.WithDecoderDicts(dictBytes)) 99 | if err != nil { 100 | return nil, err 101 | } 102 | // zst.Decoder does not properly implement a ReadCloser 103 | return &zstReadCloser{out}, nil 104 | } 105 | -------------------------------------------------------------------------------- /hack/gen-cert-chain.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -o pipefail 5 | 6 | usage () { 7 | echo "USAGE: ${0} [-d outdir] [-l chain_length]" 8 | exit 1 9 | } 10 | 11 | main () { 12 | while getopts ":d:l:h" opt ; do 13 | case ${opt} in 14 | d ) 15 | destination="${OPTARG}" 16 | ;; 17 | l ) 18 | chain_length="${OPTARG}" 19 | ;; 20 | h ) 21 | usage 22 | ;; 23 | \? ) 24 | echo "Invalid option: ${OPTARG}" 1>&2 25 | usage 26 | ;; 27 | : ) 28 | echo "Invalid option: ${OPTARG} requires an argument" 1>&2 29 | usage 30 | ;; 31 | esac 32 | done 33 | if [[ -z "${destination}" ]] ; then 34 | destination="$(pwd)/tls" 35 | fi 36 | if [[ -z "${chain_length}" ]] ; then 37 | chain_length=3 38 | fi 39 | 40 | re='^[0-9]+$' 41 | if ! [[ ${chain_length} =~ $re ]] ; then 42 | echo "error: ${chain_length} is not a number" 1>&2 43 | exit 2 44 | fi 45 | 46 | echo "Writing chain to ${destination} with a depth of ${chain_length}" 47 | mkdir -p "${destination}" 48 | 49 | set -x 50 | 51 | trap "rm -f ${destination}/*.srl ; rm -f ${destination}/*.csr" EXIT 52 | 53 | gen-ca "${destination}" 54 | fill-cert-chain "${destination}" "${chain_length}" 55 | } 56 | 57 | gen-ca () { 58 | local outdir="${1}" 59 | local keypath="${outdir}/ca.key" 60 | local csrpath="${outdir}/ca.csr" 61 | local certpath="${outdir}/ca.crt" 62 | 63 | echo "Generating CA private key" 64 | openssl genrsa -out "${keypath}" 4096 65 | echo "Generating CA CSR" 66 | openssl req -new -addext basicConstraints=CA:TRUE -sha256 -key "${keypath}" -out "${csrpath}" -subj "/CN=TEST-CA" 67 | echo "Self signing CA certificate for 10 years" 68 | openssl x509 -req -extfile <(printf "keyUsage=critical,cRLSign,digitalSignature,keyCertSign\nbasicConstraints=critical,CA:TRUE\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:kubenab.kube-system.svc,DNS:localhost") -signkey "${keypath}" -in "${csrpath}" -req -days 3650 -out "${certpath}" 69 | } 70 | 71 | fill-cert-chain() { 72 | local outdir="${1}" 73 | local chain_length="${2}" 74 | 75 | 76 | let "num_certs = ${chain_length} - 1" 77 | 78 | local signercert="${outdir}/ca.crt" 79 | local signerkey="${outdir}/ca.key" 80 | 81 | for cert_num in $(seq 1 ${num_certs}) ; do 82 | if [[ ${cert_num} == ${num_certs} ]] ; then 83 | name="leaf" 84 | else 85 | name="intermediate-${cert_num}" 86 | fi 87 | 88 | local keypath="${outdir}/${name}.key" 89 | local csrpath="${outdir}/${name}.csr" 90 | local certpath="${outdir}/${name}.crt" 91 | 92 | echo "Generating private key for ${name}" 93 | openssl genrsa -out "${keypath}" 4096 94 | echo "Generate CSR for ${name}" 95 | openssl req -new -sha256 -key "${keypath}" -out "${csrpath}" -subj "/CN=TEST-${name^^}" -addext "subjectAltName=DNS:kubenab.kube-system.svc,DNS:localhost" 96 | echo "Signing certificate for ${name}" 97 | if [[ "${name}" == "leaf" ]] ; then 98 | openssl x509 -req -extfile <(printf "keyUsage=nonRepudiation,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:kubenab.kube-system.svc,DNS:localhost") -in "${csrpath}" -CA "${signercert}" -CAkey "${signerkey}" -CAcreateserial -out "${certpath}" -days 365 -sha256 99 | else 100 | openssl x509 -req -extfile <(printf "basicConstraints=critical,CA:TRUE,pathlen:${chain_length}\nkeyUsage=critical,cRLSign,digitalSignature,keyCertSign\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:kubenab.kube-system.svc,DNS:localhost") -in "${csrpath}" -CA "${signercert}" -CAkey "${signerkey}" -CAcreateserial -out "${certpath}" -days 365 -sha256 101 | fi 102 | signercert="${certpath}" 103 | signerkey="${keypath}" 104 | done 105 | } 106 | 107 | main "${@}" -------------------------------------------------------------------------------- /pkg/util/util_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "strings" 9 | "testing" 10 | 11 | . "github.com/onsi/ginkgo" 12 | . "github.com/onsi/gomega" 13 | ) 14 | 15 | func TestUtils(t *testing.T) { 16 | RegisterFailHandler(Fail) 17 | RunSpecs(t, "Utils Suite") 18 | } 19 | 20 | var _ = Describe("Utils", func() { 21 | 22 | // GetTempDir() 23 | Describe("Get Temp Directory", func() { 24 | 25 | var ( 26 | cwd string 27 | tmpDir string 28 | err error 29 | ) 30 | 31 | JustBeforeEach(func() { 32 | tmpDir, err = GetTempDir() 33 | Expect(err).ToNot(HaveOccurred()) 34 | os.RemoveAll(tmpDir) 35 | }) 36 | 37 | Context("When configured to the default", func() { 38 | It("Should return a directory under the system default", func() { 39 | Expect(path.Dir(tmpDir)).To(Equal(os.TempDir())) 40 | }) 41 | }) 42 | 43 | // This test assumes current directory is writable 44 | Context("When overwritten with a custom path", func() { 45 | BeforeEach(func() { 46 | cwd, err = os.Getwd() 47 | Expect(err).ToNot(HaveOccurred()) 48 | TempDir = cwd 49 | }) 50 | It("Should return a temp directory under the custom path", func() { 51 | Expect(path.Dir(tmpDir)).To(Equal(cwd)) 52 | }) 53 | }) 54 | 55 | }) 56 | 57 | // CalculateSHA256Sum 58 | Describe("Calculating SHA256 Sums", func() { 59 | var ( 60 | shaSum string 61 | err error 62 | body io.ReadCloser 63 | ) 64 | 65 | const ( 66 | helloWorldSha = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" 67 | ) 68 | 69 | JustBeforeEach(func() { 70 | shaSum, err = CalculateSHA256Sum(body) 71 | }) 72 | 73 | Context("When passed the value 'hello world'", func() { 74 | BeforeEach(func() { 75 | body = ioutil.NopCloser(strings.NewReader("hello world")) 76 | }) 77 | It("Should return the correct checksum", func() { 78 | Expect(err).ToNot(HaveOccurred()) 79 | Expect(shaSum).To(Equal(helloWorldSha)) 80 | }) 81 | }) 82 | 83 | Context("When passed a closed io.Reader", func() { 84 | BeforeEach(func() { 85 | body, _, err = os.Pipe() 86 | Expect(err).ToNot(HaveOccurred()) 87 | body.Close() 88 | }) 89 | It("Should return an error", func() { 90 | Expect(err).To(HaveOccurred()) 91 | }) 92 | }) 93 | }) 94 | 95 | // IsK8sObject 96 | Describe("Detecting K8s Objects from Unmarshaled Data", func() { 97 | var ( 98 | data map[string]interface{} 99 | isK8sObject bool 100 | ) 101 | 102 | JustBeforeEach(func() { isK8sObject = IsK8sObject(data) }) 103 | 104 | Context("When passed a valid kubernetes object", func() { 105 | BeforeEach(func() { 106 | data = map[string]interface{}{ 107 | "kind": "Pod", 108 | "apiVersion": "v1", 109 | "metadata": map[string]interface{}{ 110 | "name": "test-pod", 111 | }, 112 | } 113 | }) 114 | Specify("That it is a valid object", func() { 115 | Expect(isK8sObject).To(BeTrue()) 116 | }) 117 | }) 118 | 119 | Context("When passed an invalid kubernetes object", func() { 120 | BeforeEach(func() { 121 | data = map[string]interface{}{ 122 | "hello": "world", 123 | "apiVersion": "v1", 124 | "metadata": map[string]interface{}{ 125 | "name": "invalid-pod", 126 | }, 127 | } 128 | }) 129 | Specify("That it is an invalid object", func() { 130 | Expect(isK8sObject).To(BeFalse()) 131 | }) 132 | }) 133 | }) 134 | 135 | // GenerateToken 136 | Describe("Generating Unique Tokens", func() { 137 | var ( 138 | length int 139 | token string 140 | ) 141 | JustBeforeEach(func() { token = GenerateToken(length) }) 142 | Context("When told to generate a 128 character token", func() { 143 | BeforeEach(func() { length = 128 }) 144 | It("should return a token with 128 characters", func() { 145 | Expect(len(token)).To(Equal(128)) 146 | }) 147 | }) 148 | Context("When told to generate a 256 character token", func() { 149 | BeforeEach(func() { length = 256 }) 150 | It("should return a token with 256 characters", func() { 151 | Expect(len(token)).To(Equal(256)) 152 | }) 153 | }) 154 | }) 155 | 156 | }) 157 | -------------------------------------------------------------------------------- /pkg/cache/download_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "net/http" 7 | "strings" 8 | "testing" 9 | 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | 13 | "github.com/tinyzimmer/k3p/pkg/log" 14 | ) 15 | 16 | func TestUtils(t *testing.T) { 17 | RegisterFailHandler(Fail) 18 | RunSpecs(t, "Download Cache Suite") 19 | } 20 | 21 | var mockCalled bool 22 | 23 | func mockGet(url string) (*http.Response, error) { 24 | mockCalled = true 25 | return &http.Response{ 26 | StatusCode: http.StatusOK, 27 | Body: ioutil.NopCloser(strings.NewReader("test")), 28 | }, nil 29 | } 30 | 31 | var _ = Describe("Download Cache", func() { 32 | // Send log output to the ginkgo writer 33 | log.LogWriter = GinkgoWriter 34 | 35 | // The DefaultCache should never be nil 36 | Expect(DefaultCache).ToNot(BeNil()) 37 | // NoCache should default to false 38 | Expect(NoCache).To(BeFalse()) 39 | 40 | Describe("Creating a new cache", func() { 41 | var ( 42 | cache HTTPCache 43 | ) 44 | JustBeforeEach(func() { cache = New("") }) 45 | Context("When creating a new cache from a directory", func() { 46 | It("Should not be nil", func() { 47 | Expect(cache).ToNot(BeNil()) 48 | }) 49 | }) 50 | }) 51 | 52 | Describe("Get cache directory", func() { 53 | 54 | var ( 55 | cache *httpCache 56 | cacheDir string 57 | ) 58 | 59 | JustBeforeEach(func() { 60 | cacheDir = cache.CacheDir() 61 | }) 62 | 63 | Context("With a configured http cache", func() { 64 | BeforeEach(func() { 65 | cache = &httpCache{cacheDir: "test"} 66 | }) 67 | It("Should return the configured cache directory", func() { 68 | Expect(cacheDir).To(Equal("test")) 69 | }) 70 | }) 71 | }) 72 | 73 | Describe("Clean cache directory", func() { 74 | 75 | var ( 76 | cache *httpCache 77 | err error 78 | ) 79 | 80 | JustBeforeEach(func() { err = cache.Clean() }) 81 | 82 | Context("With no cache directory configured", func() { 83 | BeforeEach(func() { 84 | cache = &httpCache{} 85 | }) 86 | It("Should return an error", func() { 87 | Expect(err).To(HaveOccurred()) 88 | }) 89 | }) 90 | 91 | Context("With a valid cache directory configured", func() { 92 | BeforeEach(func() { 93 | tmpDir, err := ioutil.TempDir("", "") 94 | Expect(err).ToNot(HaveOccurred()) 95 | cache = &httpCache{cacheDir: tmpDir} 96 | }) 97 | It("Should not return an error", func() { 98 | Expect(err).ToNot(HaveOccurred()) 99 | }) 100 | }) 101 | 102 | }) 103 | 104 | Describe("Downloading files using the cache", func() { 105 | var ( 106 | tmpDir string 107 | cache *httpCache 108 | url string 109 | body io.ReadCloser 110 | err error 111 | ) 112 | 113 | tmpDir, err = ioutil.TempDir("", "") 114 | Expect(err).ToNot(HaveOccurred()) 115 | cache = &httpCache{cacheDir: tmpDir, get: mockGet} 116 | 117 | JustBeforeEach(func() { 118 | mockCalled = false 119 | body, err = cache.Get(url) 120 | }) 121 | 122 | Context("When retrieving a non-cached URL", func() { 123 | BeforeEach(func() { url = "https://example.com" }) 124 | It("Should retrieve the body from the web and write it to the cache", func() { 125 | Expect(err).ToNot(HaveOccurred()) 126 | Expect(body).ToNot(BeNil()) 127 | Expect(mockCalled).To(BeTrue()) 128 | files, err := ioutil.ReadDir(tmpDir) 129 | Expect(err).ToNot(HaveOccurred()) 130 | Expect(len(files)).To(Equal(1)) 131 | }) 132 | }) 133 | 134 | Context("When retrieving a cached URL", func() { 135 | BeforeEach(func() { 136 | url = "https://example.com" 137 | _, err = cache.Get(url) 138 | Expect(err).ToNot(HaveOccurred()) 139 | }) 140 | It("Should retrieve the contents from the local cache", func() { 141 | Expect(err).ToNot(HaveOccurred()) 142 | Expect(body).ToNot(BeNil()) 143 | Expect(mockCalled).To(BeFalse()) 144 | }) 145 | }) 146 | 147 | Context("When caching is disabled", func() { 148 | BeforeEach(func() { 149 | NoCache = true 150 | url = "https://example.com" 151 | _, err = cache.Get(url) 152 | Expect(err).ToNot(HaveOccurred()) 153 | }) 154 | It("Should always retrieve from the web", func() { 155 | Expect(err).ToNot(HaveOccurred()) 156 | Expect(body).ToNot(BeNil()) 157 | Expect(mockCalled).To(BeTrue()) 158 | }) 159 | }) 160 | }) 161 | }) 162 | -------------------------------------------------------------------------------- /hack/fake-eula.txt: -------------------------------------------------------------------------------- 1 | The standard Lorem Ipsum passage, used since the 1500s 2 | "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. 3 | Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure 4 | dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 5 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum." 6 | 7 | Section 1.10.32 of "de Finibus Bonorum et Malorum", written by Cicero in 45 BC 8 | "Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque 9 | ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia 10 | voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. 11 | Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi 12 | tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam 13 | corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate 14 | velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?" 15 | 16 | 1914 translation by H. Rackham 17 | "But I must explain to you how all this mistaken idea of denouncing pleasure and praising pain was born and I will give you a complete 18 | account of the system, and expound the actual teachings of the great explorer of the truth, the master-builder of human happiness. 19 | No one rejects, dislikes, or avoids pleasure itself, because it is pleasure, but because those who do not know how to pursue pleasure 20 | rationally encounter consequences that are extremely painful. Nor again is there anyone who loves or pursues or desires to obtain 21 | pain of itself, because it is pain, but because occasionally circumstances occur in which toil and pain can procure him some great 22 | pleasure. To take a trivial example, which of us ever undertakes laborious physical exercise, except to obtain some advantage from it? 23 | But who has any right to find fault with a man who chooses to enjoy a pleasure that has no annoying consequences, or one who avoids a 24 | pain that produces no resultant pleasure?" 25 | 26 | Section 1.10.33 of "de Finibus Bonorum et Malorum", written by Cicero in 45 BC 27 | "At vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores 28 | et quas molestias excepturi sint occaecati cupiditate non provident, similique sunt in culpa qui officia deserunt mollitia animi, id 29 | est laborum et dolorum fuga. Et harum quidem rerum facilis est et expedita distinctio. Nam libero tempore, cum soluta nobis est eligendi 30 | optio cumque nihil impedit quo minus id quod maxime placeat facere possimus, omnis voluptas assumenda est, omnis dolor repellendus. 31 | Temporibus autem quibusdam et aut officiis debitis aut rerum necessitatibus saepe eveniet ut et voluptates repudiandae sint et molestiae 32 | non recusandae. Itaque earum rerum hic tenetur a sapiente delectus, ut aut reiciendis voluptatibus maiores alias consequatur aut 33 | perferendis doloribus asperiores repellat." 34 | 35 | 1914 translation by H. Rackham 36 | "On the other hand, we denounce with righteous indignation and dislike men who are so beguiled and demoralized by the charms of pleasure 37 | of the moment, so blinded by desire, that they cannot foresee the pain and trouble that are bound to ensue; and equal blame belongs to those 38 | who fail in their duty through weakness of will, which is the same as saying through shrinking from toil and pain. These cases are perfectly 39 | simple and easy to distinguish. In a free hour, when our power of choice is untrammelled and when nothing prevents our being able to do what 40 | we like best, every pleasure is to be welcomed and every pain avoided. But in certain circumstances and owing to the claims of duty or the 41 | obligations of business it will frequently occur that pleasures have to be repudiated and annoyances accepted. The wise man therefore always 42 | holds in these matters to this principle of selection: he rejects pleasures to secure other greater pleasures, or else he endures pains to 43 | avoid worse pains." -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "os" 8 | "time" 9 | ) 10 | 11 | // Verbose is set by the CLI flag to enable debug logging 12 | var Verbose bool 13 | 14 | // LogWriter can be overwritten by tests to suppress log output 15 | var LogWriter io.Writer = os.Stdout 16 | 17 | // Level represents a logging level 18 | type Level string 19 | 20 | // Log Levels 21 | const ( 22 | LevelInfo Level = "INFO" 23 | LevelWarning Level = "WARNING" 24 | LevelError Level = "ERROR" 25 | LevelDebug Level = "DEBUG" 26 | ) 27 | 28 | var infoLogger, warningLogger, errorLogger, debugLogger *logger 29 | 30 | const ( 31 | boldColor = "\033[1m%s\033[0m" 32 | infoColor = "\033[0;34m%s\033[0m" 33 | noticeColor = "\033[0;36m%s\033[0m" 34 | warningColor = "\033[0;33m%s\033[0m" 35 | errorColor = "\033[0;31m%s\033[0m" 36 | debugColor = "\033[0;36m%s\033[0m" 37 | ) 38 | 39 | func init() { 40 | infoLogger = &logger{ 41 | prefix: LevelInfo, 42 | color: infoColor, 43 | } 44 | warningLogger = &logger{ 45 | prefix: LevelWarning, 46 | color: warningColor, 47 | } 48 | errorLogger = &logger{ 49 | prefix: LevelError, 50 | color: errorColor, 51 | } 52 | debugLogger = &logger{ 53 | prefix: LevelDebug, 54 | color: debugColor, 55 | } 56 | } 57 | 58 | type logger struct { 59 | prefix Level 60 | color string 61 | } 62 | 63 | func (l *logger) getPrefix() string { 64 | return fmt.Sprintf(l.color, fmt.Sprintf("[%s]", l.prefix)) 65 | } 66 | 67 | const timeFormat = "2006/01/02 15:04:05" 68 | 69 | func (l *logger) getTime() string { 70 | return fmt.Sprintf(noticeColor, time.Now().Local().Format(timeFormat)) 71 | } 72 | 73 | func (l *logger) seedLine() { 74 | fmt.Fprint(LogWriter, l.getTime(), " ", l.getPrefix(), "\t") 75 | } 76 | 77 | func (l *logger) Println(args ...interface{}) { 78 | l.seedLine() 79 | line := fmt.Sprintln(args...) 80 | fmt.Fprintf(LogWriter, boldColor, line) 81 | } 82 | 83 | func (l *logger) Printf(fstr string, args ...interface{}) { 84 | l.seedLine() 85 | line := fmt.Sprintf(fstr, args...) 86 | fmt.Fprintf(LogWriter, boldColor, line) 87 | } 88 | 89 | // Info is the equivalent of a log.Println on the info logger. 90 | func Info(args ...interface{}) { 91 | infoLogger.Println(args...) 92 | } 93 | 94 | // Infof is the equivalent of a log.Printf on the info logger. 95 | func Infof(fstr string, args ...interface{}) { 96 | infoLogger.Printf(fstr, args...) 97 | } 98 | 99 | // Warning is the equivalent of a log.Println on the warning logger. 100 | func Warning(args ...interface{}) { 101 | warningLogger.Println(args...) 102 | } 103 | 104 | // Warningf is the equivalent of a log.Printf on the warning logger. 105 | func Warningf(fstr string, args ...interface{}) { 106 | warningLogger.Printf(fstr, args...) 107 | } 108 | 109 | // Error is the equivalent of a log.Println on the error logger. 110 | func Error(args ...interface{}) { 111 | errorLogger.Println(args...) 112 | } 113 | 114 | // Errorf is the equivalent of a log.Printf on the error logger. 115 | func Errorf(fstr string, args ...interface{}) { 116 | errorLogger.Printf(fstr, args...) 117 | } 118 | 119 | // Fatal is a convenience wrapper around logging to the error logger 120 | // and exiting imediately. 121 | func Fatal(args ...interface{}) { 122 | Error("Fatal exception ocurred") 123 | Error(args...) 124 | os.Exit(1) 125 | } 126 | 127 | // Debug is the equivalent of a log.Println on the debug logger. 128 | func Debug(args ...interface{}) { 129 | if !Verbose { 130 | return 131 | } 132 | debugLogger.Println(args...) 133 | } 134 | 135 | // Debugf is the equivalent of a log.Printf on the debug logger. 136 | func Debugf(fstr string, args ...interface{}) { 137 | if !Verbose { 138 | return 139 | } 140 | debugLogger.Printf(fstr, args...) 141 | } 142 | 143 | func getLoggerForLevel(level Level) *logger { 144 | switch level { 145 | case LevelInfo: 146 | return infoLogger 147 | case LevelWarning: 148 | return warningLogger 149 | case LevelError: 150 | return errorLogger 151 | case LevelDebug: 152 | return debugLogger 153 | } 154 | return &logger{prefix: level, color: infoColor} 155 | } 156 | 157 | // LevelReader is a convenience method for tailing the contents of a reader 158 | // to the logger specified by the given level. 159 | func LevelReader(level Level, rdr io.Reader) { 160 | l := getLoggerForLevel(level) 161 | scanner := bufio.NewScanner(rdr) 162 | for scanner.Scan() { 163 | text := scanner.Text() 164 | if level == LevelDebug && !Verbose { 165 | continue // This function needs to block even if not logging verbosely 166 | } 167 | l.Println(text) 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /pkg/types/package_meta.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | ) 7 | 8 | // PackageMeta represents metadata included with a package. 9 | type PackageMeta struct { 10 | // The version of this manifest, only v1 currently 11 | MetaVersion string `json:"apiVersion,omitempty"` 12 | // The name of the package 13 | Name string `json:"name,omitempty"` 14 | // The version of the package 15 | Version string `json:"version,omitempty"` 16 | // The K3s version inside the package 17 | K3sVersion string `json:"k3sVersion,omitempty"` 18 | // The architecture the package was built for 19 | Arch string `json:"arch,omitempty"` 20 | // The format with which images were bundles in the archive. 21 | ImageBundleFormat ImageBundleFormat `json:"imageBundleFormat,omitempty"` 22 | // A listing of the contents of the package 23 | Manifest *Manifest `json:"manifest,omitempty"` 24 | // A configuration containing installation variables 25 | PackageConfig *PackageConfig `json:"config,omitempty"` 26 | // The raw, untemplated package config 27 | PackageConfigRaw []byte `json:"configRaw,omitempty"` 28 | } 29 | 30 | // DeepCopy creates a copy of this PackageMeta instance. 31 | // TODO: DeepCopy functions need to be generated 32 | func (p *PackageMeta) DeepCopy() *PackageMeta { 33 | meta := &PackageMeta{ 34 | MetaVersion: p.MetaVersion, 35 | Name: p.Name, 36 | Version: p.Version, 37 | K3sVersion: p.K3sVersion, 38 | Arch: p.Arch, 39 | ImageBundleFormat: p.ImageBundleFormat, 40 | PackageConfigRaw: make([]byte, len(p.PackageConfigRaw)), 41 | } 42 | copy(meta.PackageConfigRaw, p.PackageConfigRaw) 43 | if p.Manifest != nil { 44 | meta.Manifest = p.Manifest.DeepCopy() 45 | } 46 | if p.PackageConfig != nil { 47 | meta.PackageConfig = p.PackageConfig.DeepCopy() 48 | } 49 | return meta 50 | } 51 | 52 | // Sanitize will iterate the PackageConfig and convert any `map[interface{}]interface{}` 53 | // to `map[string]interface{}`. This is required for serializing meta until I find a better 54 | // way to deal with helm values. For convenience, the pointer to the PackageMeta is returned. 55 | func (p *PackageMeta) Sanitize() *PackageMeta { 56 | if p.PackageConfig == nil { 57 | return p 58 | } 59 | newHelmValues := make(map[string]interface{}) 60 | for key, value := range p.PackageConfig.HelmValues { 61 | newHelmValues[key] = sanitizeValue(value) 62 | } 63 | p.PackageConfig.HelmValues = newHelmValues 64 | return p 65 | } 66 | 67 | func sanitizeValue(val interface{}) interface{} { 68 | switch reflect.TypeOf(val).Kind() { 69 | case reflect.Map: 70 | if m, ok := val.(map[interface{}]interface{}); ok { 71 | newMap := make(map[string]interface{}) 72 | for k, v := range m { 73 | kStr := fmt.Sprintf("%v", k) 74 | newMap[kStr] = sanitizeValue(v) 75 | } 76 | return newMap 77 | } 78 | if m, ok := val.(map[string]interface{}); ok { 79 | // if the keys are already strings, we still need to descend 80 | newMap := make(map[string]interface{}) 81 | for k, v := range m { 82 | newMap[k] = sanitizeValue(v) 83 | } 84 | return newMap 85 | } 86 | // otherwise just return the regular map, but this may not catch 87 | // all cases yet 88 | return val 89 | default: 90 | return val 91 | } 92 | } 93 | 94 | // GetName returns the name of the package. 95 | func (p *PackageMeta) GetName() string { return p.Name } 96 | 97 | // GetVersion returns the version of the package. 98 | func (p *PackageMeta) GetVersion() string { return p.Version } 99 | 100 | // GetK3sVersion returns the K3s version for the package. 101 | func (p *PackageMeta) GetK3sVersion() string { return p.K3sVersion } 102 | 103 | // GetArch returns the CPU architecture fo rthe package. 104 | func (p *PackageMeta) GetArch() string { return p.Arch } 105 | 106 | // GetManifest returns the manifest of the package. 107 | func (p *PackageMeta) GetManifest() *Manifest { return p.Manifest } 108 | 109 | // GetPackageConfig returns the package config if of the package or nil if there is none. 110 | func (p *PackageMeta) GetPackageConfig() *PackageConfig { return p.PackageConfig } 111 | 112 | // GetRegistryImageName returns the name that would have been used for a container image 113 | // containing the registry contents. 114 | // TODO: Needing to keep this logic here and BuildRegistryOptions is not a good design probably. 115 | func (p *PackageMeta) GetRegistryImageName() string { 116 | return fmt.Sprintf("%s-private-registry-data:%s", p.Name, p.Version) 117 | } 118 | 119 | // NewEmptyMeta returns a new empty PackageMeta instance. 120 | func NewEmptyMeta() *PackageMeta { 121 | return &PackageMeta{Manifest: NewEmptyManifest()} 122 | } 123 | -------------------------------------------------------------------------------- /pkg/parser/obj_parse.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "io/ioutil" 5 | "strings" 6 | 7 | "github.com/tinyzimmer/k3p/pkg/log" 8 | "github.com/tinyzimmer/k3p/pkg/util" 9 | 10 | appsv1 "k8s.io/api/apps/v1" 11 | appsv1beta1 "k8s.io/api/apps/v1beta1" 12 | batchv1 "k8s.io/api/batch/v1" 13 | batchv1betav1 "k8s.io/api/batch/v1beta1" 14 | corev1 "k8s.io/api/core/v1" 15 | "k8s.io/apimachinery/pkg/runtime" 16 | ) 17 | 18 | func (p *ManifestParser) parseFileForImages(file string, renderVars map[string]string) ([]string, error) { 19 | images := make([]string, 0) 20 | data, err := ioutil.ReadFile(file) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | if len(renderVars) > 0 { 26 | data, err = util.RenderBody(data, renderVars) 27 | if err != nil { 28 | return nil, err 29 | } 30 | } 31 | 32 | // iterate all the yaml objects in the file 33 | rawYamls := strings.Split(string(data), "---") 34 | for _, raw := range rawYamls { 35 | // Check if this is empty space 36 | if strings.TrimSpace(raw) == "" { 37 | continue 38 | } 39 | // Decode the object 40 | obj, err := p.Decode([]byte(raw)) 41 | if err != nil { 42 | log.Debugf("Skipping invalid kubernetes object in %q: %s\n", file, err.Error()) 43 | continue 44 | } 45 | // Append any images to the local images to be downloaded 46 | if objImgs := parseObjectForImages(obj); len(objImgs) > 0 { 47 | images = appendIfMissing(images, objImgs...) 48 | } 49 | } 50 | return images, nil 51 | } 52 | 53 | func parseObjectForImages(obj runtime.Object) []string { 54 | images := make([]string, 0) 55 | 56 | gvk := obj.GetObjectKind().GroupVersionKind() 57 | 58 | switch gvk.Kind { 59 | case "Pod": 60 | pod := obj.(*corev1.Pod) 61 | log.Info("Found Pod:", pod.GetName()) 62 | if imgs := parseImagesFromContainers(pod.Spec.Containers); len(imgs) > 0 { 63 | images = append(images, imgs...) 64 | } 65 | case "DaemonSet": 66 | daemonset, ok := obj.(*appsv1.DaemonSet) 67 | if !ok { 68 | log.Info("Skipping non apps/v1 DaemonSet") 69 | return images 70 | } 71 | log.Info("Found DaemonSet:", daemonset.GetName()) 72 | if imgs := parseImagesFromContainers(daemonset.Spec.Template.Spec.Containers); len(imgs) > 0 { 73 | images = append(images, imgs...) 74 | } 75 | case "Deployment": // only supports appsv1 and v1beta1 for now 76 | switch gvk.Version { 77 | case "v1": 78 | deployment := obj.(*appsv1.Deployment) 79 | log.Info("Found appsv1 Deployment:", deployment.GetName()) 80 | if imgs := parseImagesFromContainers(deployment.Spec.Template.Spec.Containers); len(imgs) > 0 { 81 | images = append(images, imgs...) 82 | } 83 | case "v1beta1": 84 | deployment := obj.(*appsv1beta1.Deployment) 85 | log.Info("Found appsv1beta1 Deployment:", deployment.GetName()) 86 | if imgs := parseImagesFromContainers(deployment.Spec.Template.Spec.Containers); len(imgs) > 0 { 87 | images = append(images, imgs...) 88 | } 89 | default: 90 | log.Info("Skipping non apps/v1 or apps/v1beta1 Deployment object") 91 | } 92 | case "StatefulSet": // only supports apps/v1 93 | ss, ok := obj.(*appsv1.StatefulSet) 94 | if !ok { 95 | log.Info("Skipping non apps/v1 StatefulSet object") 96 | return images 97 | } 98 | log.Info("Found StatefulSet:", ss.GetName()) 99 | if imgs := parseImagesFromContainers(ss.Spec.Template.Spec.Containers); len(imgs) > 0 { 100 | images = append(images, imgs...) 101 | } 102 | case "Job": // only supports batch/v1 103 | job, ok := obj.(*batchv1.Job) 104 | if !ok { 105 | log.Info("Skipping non batch/v1 Job object") 106 | return images 107 | } 108 | log.Info("Found Job:", job.GetName()) 109 | if imgs := parseImagesFromContainers(job.Spec.Template.Spec.Containers); len(imgs) > 0 { 110 | images = append(images, imgs...) 111 | } 112 | case "CronJob": // only supports batch/v1beta1 113 | job, ok := obj.(*batchv1betav1.CronJob) 114 | if !ok { 115 | log.Info("Skipping non batch/v1betav1 CronJob object") 116 | return images 117 | } 118 | log.Info("Found CronJob:", job.GetName()) 119 | if imgs := parseImagesFromContainers(job.Spec.JobTemplate.Spec.Template.Spec.Containers); len(imgs) > 0 { 120 | images = append(images, imgs...) 121 | } 122 | default: 123 | log.Debug("Skipping non-container based object:", gvk.Kind) // TODO: verbose logging 124 | } 125 | 126 | return images 127 | } 128 | 129 | func parseImagesFromContainers(containers []corev1.Container) []string { 130 | images := make([]string, 0) 131 | for _, container := range containers { 132 | if container.Image != "" { 133 | log.Debug("Found container image:", container.Image) 134 | images = append(images, container.Image) 135 | } 136 | } 137 | return images 138 | } 139 | -------------------------------------------------------------------------------- /pkg/types/docker.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | dockertypes "github.com/docker/docker/api/types" 9 | "github.com/docker/docker/api/types/filters" 10 | ) 11 | 12 | const rancherRepo = "rancher/k3s" 13 | 14 | // DockerNodeOptions are options for configuring a docker container 15 | // as a k3s node. 16 | type DockerNodeOptions struct { 17 | // The cluster options associated with this node 18 | ClusterOptions *DockerClusterOptions 19 | // The index for this node in the cluster 20 | NodeIndex int 21 | // The role of the node 22 | NodeRole K3sRole 23 | } 24 | 25 | // DockerClusterOptions represent options for building a cluster backed by docker 26 | // containers. 27 | type DockerClusterOptions struct { 28 | // The name of the cluster 29 | ClusterName string 30 | // The version of the k3s image to pull for this node 31 | K3sVersion string 32 | // The number of servers and agents to run in the cluster 33 | Servers, Agents int 34 | // Additional port mappings to apply to the leader node 35 | PortMappings []string 36 | } 37 | 38 | // DockerClusterFilters returns the filters for matching all components of a given cluster. 39 | func DockerClusterFilters(name string) filters.Args { 40 | return filters.NewArgs( 41 | filters.Arg("label", fmt.Sprintf("%s=true", K3pManagedDockerLabel)), 42 | filters.Arg("label", fmt.Sprintf("%s=%s", K3pDockerClusterLabel, name)), 43 | ) 44 | } 45 | 46 | // AllDockerClusterFilters returns the filters for listining all docker clusters installed 47 | // to the system. 48 | func AllDockerClusterFilters() filters.Args { 49 | return filters.NewArgs(filters.Arg("label", fmt.Sprintf("%s=true", K3pManagedDockerLabel))) 50 | } 51 | 52 | // DockerOptionsFromContainer converts a container spec to docker node options. 53 | func DockerOptionsFromContainer(container dockertypes.Container) *DockerNodeOptions { 54 | opts := &DockerNodeOptions{ClusterOptions: &DockerClusterOptions{}} 55 | 56 | opts.ClusterOptions.ClusterName = container.Labels[K3pDockerClusterLabel] 57 | opts.NodeRole = K3sRole(container.Labels[K3pDockerNodeRoleLabel]) 58 | 59 | versFields := strings.Split(container.Image, ":") 60 | opts.ClusterOptions.K3sVersion = versFields[len(versFields)-1] 61 | 62 | nameFields := strings.Split(container.Names[0], "-") 63 | idx, err := strconv.Atoi(nameFields[len(nameFields)-1]) 64 | if err == nil { 65 | opts.NodeIndex = idx 66 | } 67 | 68 | return opts 69 | } 70 | 71 | // GetLabels returns the labels for the node represented by these options. 72 | func (d *DockerClusterOptions) GetLabels() map[string]string { 73 | return map[string]string{ 74 | K3pManagedDockerLabel: "true", 75 | K3pDockerClusterLabel: d.ClusterName, 76 | } 77 | } 78 | 79 | // GetK3sImage returns the K3s image for these options 80 | func (d *DockerNodeOptions) GetK3sImage() string { 81 | if d.NodeRole == K3sRoleLoadBalancer { 82 | return "rancher/k3d-proxy:latest" 83 | } 84 | return fmt.Sprintf("%s:%s", rancherRepo, strings.Replace(d.ClusterOptions.K3sVersion, "+", "-", -1)) 85 | } 86 | 87 | // GetNodeName returns the name for the node represented by these options. 88 | func (d *DockerNodeOptions) GetNodeName() string { 89 | nodeRole := K3sRoleServer 90 | if d.NodeRole != "" { 91 | nodeRole = d.NodeRole 92 | } 93 | if nodeRole == K3sRoleLoadBalancer { 94 | return fmt.Sprintf("%s-serverlb", strings.Replace(d.ClusterOptions.ClusterName, "_", "-", -1)) 95 | } 96 | return fmt.Sprintf( 97 | "%s-%s-%s", 98 | strings.Replace(d.ClusterOptions.ClusterName, "_", "-", -1), 99 | string(nodeRole), 100 | strconv.Itoa(d.NodeIndex), 101 | ) 102 | } 103 | 104 | // GetLabels returns the labels for the node represented by these options. 105 | func (d *DockerNodeOptions) GetLabels() map[string]string { 106 | labels := d.ClusterOptions.GetLabels() 107 | labels[K3pDockerNodeNameLabel] = d.GetNodeName() 108 | nodeRole := string(K3sRoleServer) 109 | if d.NodeRole != "" { 110 | nodeRole = string(d.NodeRole) 111 | } 112 | labels[K3pDockerNodeRoleLabel] = nodeRole 113 | return labels 114 | } 115 | 116 | // GetComponentLabels returns the labels for a specific component represented by these options. 117 | func (d *DockerNodeOptions) GetComponentLabels(component string) map[string]string { 118 | labels := d.GetLabels() 119 | labels["component"] = component 120 | return labels 121 | } 122 | 123 | // GetFilters returns the docker filters for the nodes represented by these options. 124 | func (d *DockerNodeOptions) GetFilters() filters.Args { 125 | args := []filters.KeyValuePair{} 126 | for k, v := range d.GetLabels() { 127 | args = append(args, filters.Arg("label", fmt.Sprintf("%s=%s", k, v))) 128 | } 129 | return filters.NewArgs(args...) 130 | } 131 | -------------------------------------------------------------------------------- /pkg/cluster/node/remote.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "net" 8 | "path" 9 | "strconv" 10 | 11 | "github.com/bramvdbogaerde/go-scp" 12 | 13 | "github.com/tinyzimmer/k3p/pkg/log" 14 | "github.com/tinyzimmer/k3p/pkg/types" 15 | "golang.org/x/crypto/ssh" 16 | ) 17 | 18 | // Connect will connect to a node over SSH with the given options. 19 | func Connect(opts *types.NodeConnectOptions) (types.Node, error) { 20 | var err error 21 | n := &remoteNode{remoteAddr: opts.Address} 22 | n.client, err = getSSHClient(opts) 23 | return n, err 24 | } 25 | 26 | type remoteNode struct { 27 | client *ssh.Client 28 | remoteAddr string 29 | } 30 | 31 | func (n *remoteNode) GetType() types.NodeType { return types.NodeRemote } 32 | 33 | func (n *remoteNode) scpClient() (*scp.Client, error) { 34 | scpClient, err := scp.NewClientBySSH(n.client) 35 | if err != nil { 36 | return nil, err 37 | } 38 | scpClient.RemoteBinary = "sudo scp" 39 | return &scpClient, nil 40 | } 41 | 42 | type remoteReadCloser struct { 43 | sess *ssh.Session 44 | pipe io.Reader 45 | } 46 | 47 | func (r *remoteReadCloser) Read(p []byte) (int, error) { return r.pipe.Read(p) } 48 | 49 | func (r *remoteReadCloser) Close() error { 50 | if err := r.sess.Wait(); err != nil { 51 | return err 52 | } 53 | return r.sess.Close() 54 | } 55 | 56 | func (n *remoteNode) GetFile(path string) (io.ReadCloser, error) { 57 | sess, err := n.client.NewSession() 58 | if err != nil { 59 | return nil, err 60 | } 61 | outPipe, err := sess.StdoutPipe() 62 | if err != nil { 63 | return nil, err 64 | } 65 | remoteRdr := &remoteReadCloser{sess: sess, pipe: outPipe} 66 | cmd := fmt.Sprintf("sudo cat %q", path) 67 | log.Debugf("Running command on %s: %s\n", n.remoteAddr, cmd) 68 | if err := sess.Start(cmd); err != nil { 69 | if cerr := sess.Close(); cerr != nil { 70 | log.Error("Unexpected error while closing failed ssh get file:", cerr) 71 | } 72 | return nil, err 73 | } 74 | return remoteRdr, nil 75 | } 76 | 77 | func (n *remoteNode) WriteFile(rdr io.ReadCloser, destination string, mode string, size int64) error { 78 | if err := n.MkdirAll(path.Dir(destination)); err != nil { 79 | return err 80 | } 81 | scpClient, err := n.scpClient() 82 | if err != nil { 83 | return err 84 | } 85 | defer scpClient.Close() 86 | defer rdr.Close() 87 | log.Debugf("Sending %d bytes of %q to %q on %s and setting mode to %s\n", size, path.Base(destination), destination, n.remoteAddr, mode) 88 | return scpClient.Copy(rdr, destination, mode, size) 89 | } 90 | 91 | func (n *remoteNode) MkdirAll(dir string) error { 92 | sess, err := n.client.NewSession() 93 | if err != nil { 94 | return err 95 | } 96 | defer sess.Close() 97 | cmd := fmt.Sprintf("sudo mkdir -p %s", dir) 98 | log.Debugf("Running command on %s: %s\n", n.remoteAddr, cmd) 99 | return sess.Run(cmd) 100 | } 101 | 102 | func (n *remoteNode) Execute(opts *types.ExecuteOptions) error { 103 | sess, err := n.client.NewSession() 104 | if err != nil { 105 | return err 106 | } 107 | outPipe, err := sess.StdoutPipe() 108 | if err != nil { 109 | return err 110 | } 111 | errPipe, err := sess.StderrPipe() 112 | if err != nil { 113 | return err 114 | } 115 | cmd := buildCmdFromExecOpts(opts) 116 | log.Debugf("Executing command on %s: %s\n", n.remoteAddr, redactSecrets(cmd, opts.Secrets)) 117 | go log.LevelReader(log.LevelInfo, outPipe) 118 | go log.LevelReader(log.LevelDebug, errPipe) 119 | return sess.Run(cmd) 120 | } 121 | 122 | func (n *remoteNode) GetK3sAddress() (string, error) { 123 | // the address is assumed to be the one we connected on (TODO: fix probably) 124 | return n.remoteAddr, nil 125 | } 126 | 127 | func (n *remoteNode) Close() error { return n.client.Close() } 128 | 129 | func getSSHClient(opts *types.NodeConnectOptions) (*ssh.Client, error) { 130 | log.Debug("Using SSH user:", opts.SSHUser) 131 | config := &ssh.ClientConfig{ 132 | User: opts.SSHUser, 133 | Auth: make([]ssh.AuthMethod, 0), 134 | HostKeyCallback: ssh.InsecureIgnoreHostKey(), // TODO: obviously this should be reconsidered 135 | } 136 | config.Config.SetDefaults() 137 | if opts.SSHPassword != "" { 138 | log.Debug("Using SSH password authentication") 139 | config.Auth = append(config.Auth, ssh.Password(opts.SSHPassword)) 140 | } 141 | if opts.SSHKeyFile != "" { 142 | log.Debug("Using SSH pubkey authentication") 143 | log.Debugf("Loading SSH key from %q\n", opts.SSHKeyFile) 144 | keyBytes, err := ioutil.ReadFile(opts.SSHKeyFile) 145 | if err != nil { 146 | return nil, err 147 | } 148 | key, err := ssh.ParsePrivateKey(keyBytes) 149 | if err != nil { 150 | return nil, err 151 | } 152 | config.Auth = append(config.Auth, ssh.PublicKeys(key)) 153 | } 154 | addr := net.JoinHostPort(opts.Address, strconv.Itoa(opts.SSHPort)) 155 | log.Debugf("Creating SSH connection with %s over TCP\n", addr) 156 | return ssh.Dial("tcp", addr, config) 157 | } 158 | -------------------------------------------------------------------------------- /pkg/util/ip_util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/binary" 7 | "encoding/hex" 8 | "errors" 9 | "fmt" 10 | "math/big" 11 | "net" 12 | "os" 13 | "strings" 14 | 15 | "github.com/mitchellh/go-ps" 16 | "github.com/tinyzimmer/k3p/pkg/log" 17 | ) 18 | 19 | // RevBytes will reverse the given byte slice. 20 | func RevBytes(b []byte) []byte { 21 | for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 { 22 | b[i], b[j] = b[j], b[i] 23 | } 24 | return b 25 | } 26 | 27 | // RevIPv4 will reverse an IPv4 address 28 | func RevIPv4(ip net.IP) net.IP { 29 | rev := RevBytes(ip.To4()) 30 | return net.IPv4(rev[0], rev[1], rev[2], rev[3]) 31 | } 32 | 33 | // ReversedHexToIPv4 decodes the given hex representation of a reversed 34 | // ip address to an IP object. 35 | func ReversedHexToIPv4(h string) (net.IP, error) { 36 | ipBytes, err := hex.DecodeString(h) 37 | if err != nil { 38 | return nil, err 39 | } 40 | if len(ipBytes) != 4 { 41 | return nil, fmt.Errorf("%s is not a valid ip hex string", h) 42 | } 43 | rev := RevBytes(ipBytes) 44 | return net.IPv4(rev[0], rev[1], rev[2], rev[3]), nil 45 | } 46 | 47 | // IPv4ToReverseHex reverses the given IP address string and encodes it to 48 | // hexadecimal. 49 | func IPv4ToReverseHex(ip net.IP) string { 50 | return Pack32BinaryIPv4(RevIPv4(ip)) 51 | } 52 | 53 | // Pack32BinaryIPv4 will pack the given IP address string to 32-bit hexadecimal format. 54 | func Pack32BinaryIPv4(ip net.IP) string { 55 | ipv4Decimal := IPv4ToInt(ip) 56 | 57 | buf := new(bytes.Buffer) 58 | err := binary.Write(buf, binary.BigEndian, uint32(ipv4Decimal)) 59 | 60 | if err != nil { 61 | fmt.Println("Unable to write to buffer:", err) 62 | } 63 | 64 | // present in hexadecimal format 65 | result := fmt.Sprintf("%x", buf.Bytes()) 66 | return strings.ToUpper(result) 67 | } 68 | 69 | // IPv4ToInt produces the decimal representation of the given IP address. 70 | func IPv4ToInt(ip net.IP) int64 { 71 | ipv4Int := big.NewInt(0) 72 | ipv4Int.SetBytes(ip.To4()) 73 | return ipv4Int.Int64() 74 | } 75 | 76 | // GetPIDByName returns the first process ID that matches the given name. 77 | func GetPIDByName(name string) (int, error) { 78 | procs, err := ps.Processes() 79 | if err != nil { 80 | return 0, err 81 | } 82 | for _, proc := range procs { 83 | if proc.Executable() == name { 84 | return proc.Pid(), nil 85 | } 86 | } 87 | return 0, fmt.Errorf("No process found for %s", name) 88 | } 89 | 90 | // GetNonLoopbackAddresses returns a list of the non-loopback IP addresses 91 | // configured on the local machine. 92 | func GetNonLoopbackAddresses() ([]net.IP, error) { 93 | possibleAddrs := make([]net.IP, 0) 94 | addrs, err := net.InterfaceAddrs() 95 | if err != nil { 96 | return nil, err 97 | } 98 | for _, a := range addrs { 99 | var ip net.IP 100 | switch v := a.(type) { 101 | case *net.IPNet: 102 | ip = v.IP 103 | case *net.IPAddr: 104 | ip = v.IP 105 | } 106 | if !ip.IsLoopback() { 107 | possibleAddrs = append(possibleAddrs, ip) 108 | } 109 | } 110 | return possibleAddrs, nil 111 | } 112 | 113 | // GetExternalAddressForProcess attempts to find the external address that a process 114 | // is listening on. 115 | func GetExternalAddressForProcess(name string) (net.IP, error) { 116 | possibleAddrs, err := GetNonLoopbackAddresses() 117 | if err != nil { 118 | return nil, err 119 | } 120 | log.Debug("Possible external addresses:", possibleAddrs) 121 | pid, err := GetPIDByName(name) 122 | if err != nil { 123 | return nil, err 124 | } 125 | p := fmt.Sprintf("/proc/%d/net/tcp", pid) 126 | log.Debugf("Scanning %q for remote listener\n", p) 127 | f, err := os.Open(p) 128 | if err != nil { 129 | return nil, err 130 | } 131 | defer f.Close() 132 | scanner := bufio.NewScanner(f) 133 | scanner.Scan() // skip first line 134 | for scanner.Scan() { 135 | text := scanner.Text() 136 | fields := strings.Fields(text) 137 | if len(fields) < 3 { 138 | return nil, errors.New("Unexpected error reading proc file, line has less than 3 fields") 139 | } 140 | remoteAddrRaw := fields[2] 141 | spl := strings.Split(remoteAddrRaw, ":") 142 | if len(spl) < 2 { 143 | return nil, errors.New("Unexpected error reading proc file, addr doesn't have two parts") 144 | } 145 | addrHex := spl[0] 146 | if procLocalhost == addrHex { 147 | continue 148 | } 149 | if isPossibleAddr(possibleAddrs, addrHex) { 150 | log.Debugf("%s appears to be listening on addr hex %s\n", name, addrHex) 151 | return ReversedHexToIPv4(addrHex) 152 | } 153 | } 154 | return nil, fmt.Errorf("Could not find an external address for %s", name) 155 | } 156 | 157 | var procLocalhost = Pack32BinaryIPv4(net.IPv4(1, 0, 0, 127)) 158 | 159 | func isPossibleAddr(possible []net.IP, addrHex string) bool { 160 | for _, p := range possible { 161 | if ip4 := p.To4(); ip4 != nil { 162 | if IPv4ToReverseHex(ip4) == addrHex { 163 | return true 164 | } 165 | } 166 | } 167 | return false 168 | } 169 | -------------------------------------------------------------------------------- /pkg/types/constants.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // VersionLatest is a string signaling that the latest version should be retrieved for k3s. 4 | const VersionLatest string = "latest" 5 | 6 | // ManifestMetaFile is the name used when writing the version information to an archive. 7 | const ManifestMetaFile = "manifest.json" 8 | 9 | // ManifestEULAFile is the name used when archiving an EULA. 10 | const ManifestEULAFile = "EULA.txt" 11 | 12 | // ManifestUserImagesFile is the name of the tarball where detected images are stored in an archive. 13 | const ManifestUserImagesFile = "manifest-images.tar" 14 | 15 | // K3sRootConfigDir is the root directory where k3s assets are stored 16 | const K3sRootConfigDir = "/var/lib/rancher/k3s" 17 | 18 | // ServerTokenFile is the file where the secret token is written for joining 19 | // new control-plane instances in an HA setup. 20 | const ServerTokenFile = "/var/lib/rancher/k3s/server/server-token" 21 | 22 | // AgentTokenFile is the file where the secret token is written for joining 23 | // new agents to the cluster 24 | const AgentTokenFile = "/var/lib/rancher/k3s/server/node-token" 25 | 26 | // InstalledPackageFile is the file where the original tarball is copied 27 | // during the installation. 28 | const InstalledPackageFile = "/var/lib/rancher/k3s/data/k3p-package.tar" 29 | 30 | // InstalledConfigFile is the file where the variables used at installation are stored. 31 | const InstalledConfigFile = "/var/lib/rancher/k3s/data/k3p-config.json" 32 | 33 | // K3sManifestsDir is the directory where manifests are installed for k3s to pre-load on boot. 34 | const K3sManifestsDir = "/var/lib/rancher/k3s/server/manifests" 35 | 36 | // K3sImagesDir is the directory where images are pre-loaded on a server or agent. 37 | const K3sImagesDir = "/var/lib/rancher/k3s/agent/images" 38 | 39 | // K3sStaticDir is the directory where static content can be served from the k8s api. 40 | const K3sStaticDir = "/var/lib/rancher/k3s/server/static/k3p" 41 | 42 | // K3sScriptsDir is the directory where scripts are installed to the system. 43 | const K3sScriptsDir = "/usr/local/bin/k3p-scripts" 44 | 45 | // K3sBinDir is the directory where binaries are installed to the system. 46 | const K3sBinDir = "/usr/local/bin" 47 | 48 | // K3sEtcDir is the directory where configuration files are stored for k3s. 49 | const K3sEtcDir = "/etc/rancher/k3s" 50 | 51 | // K3sRegistriesYamlPath is the path where the k3s containerd configuration is stored. 52 | const K3sRegistriesYamlPath = "/etc/rancher/k3s/registries.yaml" 53 | 54 | // K3sKubeconfig is the path where the admin kubeconfig is stored on the system. 55 | const K3sKubeconfig = "/etc/rancher/k3s/k3s.yaml" 56 | 57 | // K3sInternalIPLabel is the label K3s uses for the internal IP of a node. 58 | const K3sInternalIPLabel = "k3s.io/internal-ip" 59 | 60 | // K3pManagedDockerLabel is the label placed on resources to mark that they were created by k3p. 61 | const K3pManagedDockerLabel = "k3p.io/managed" 62 | 63 | // K3pDockerClusterLabel is the label placed on k3p docker assets containing the cluster name. 64 | const K3pDockerClusterLabel = "k3p.io/cluster-name" 65 | 66 | // K3pDockerNodeNameLabel is the label placed on k3p docker assets containing the node name. 67 | const K3pDockerNodeNameLabel = "k3p.io/node-name" 68 | 69 | // K3pDockerNodeRoleLabel is the label where the node role is placed. 70 | const K3pDockerNodeRoleLabel = "k3p.io/node-role" 71 | 72 | // DefaultRegistryPort is the default node port used when a package includes a private registry. 73 | const DefaultRegistryPort = 30100 74 | 75 | // K3sRole represents the different roles a machine can take in the cluster 76 | type K3sRole string 77 | 78 | const ( 79 | // K3sRoleServer represents a server instance in the control-plane 80 | K3sRoleServer K3sRole = "server" 81 | // K3sRoleAgent represents a worker node instance 82 | K3sRoleAgent K3sRole = "agent" 83 | // K3sRoleLoadBalancer is a special role used for running packages in docker containers 84 | K3sRoleLoadBalancer K3sRole = "loadbalancer" 85 | ) 86 | 87 | // ArtifactType declares a type of artifact to be included in a bundle. 88 | type ArtifactType string 89 | 90 | const ( 91 | // ArtifactBin represents a binary artifact. 92 | ArtifactBin ArtifactType = "bin" 93 | // ArtifactImages represents a container image artifact. 94 | ArtifactImages ArtifactType = "images" 95 | // ArtifactScript represents a script artifact. 96 | ArtifactScript ArtifactType = "script" 97 | // ArtifactManifest represents a kubernetes manifest artifact. 98 | ArtifactManifest ArtifactType = "manifest" 99 | // ArtifactStatic represents static content to be hosted by the api server. 100 | ArtifactStatic ArtifactType = "static" 101 | // ArtifactEULA represents an End User License Agreement. 102 | ArtifactEULA ArtifactType = "eula" 103 | // ArtifactEtc is an artifact to be placed in /etc/rancher/k3s. 104 | ArtifactEtc ArtifactType = "etc" 105 | ) 106 | 107 | // ImageBundleFormat declares how the images were bundled in a package. Currently 108 | // either via raw tar balls, or a pre-loaded private registry. 109 | type ImageBundleFormat string 110 | 111 | const ( 112 | // ImageBundleTar represents raw image tarballs. 113 | ImageBundleTar ImageBundleFormat = "raw" 114 | // ImageBundleRegistry represents a pre-loaded private registry. 115 | ImageBundleRegistry ImageBundleFormat = "registry" 116 | ) 117 | -------------------------------------------------------------------------------- /pkg/cache/download.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "os" 10 | "os/user" 11 | "path" 12 | "strings" 13 | "time" 14 | 15 | "github.com/tinyzimmer/k3p/pkg/log" 16 | "github.com/tinyzimmer/k3p/pkg/util" 17 | ) 18 | 19 | // NoCache can be set by a CLI flag to signal that fresh copies should be downloaded 20 | // for every request. 21 | var NoCache bool 22 | 23 | // DefaultCache is the default http cache configured at init. It should be used for most 24 | // operations. 25 | var DefaultCache HTTPCache 26 | 27 | func init() { 28 | cache := &httpCache{get: http.Get} 29 | defer func() { DefaultCache = cache }() 30 | usr, err := user.Current() 31 | if err != nil { 32 | log.Error(err) 33 | return 34 | } 35 | cacheDir := path.Join(usr.HomeDir, ".k3p", "cache") 36 | if err := os.MkdirAll(cacheDir, 0755); err != nil { 37 | log.Error(err) 38 | return 39 | } 40 | cache.cacheDir = cacheDir 41 | } 42 | 43 | // HTTPCache is an interface for retrieving files from the internet while 44 | // maintaining a local cache of received objects for use across CLI 45 | // invocations. It is a very basic implementation that can be refactored 46 | // in the future. 47 | type HTTPCache interface { 48 | // CacheDir returns the current cache directory. 49 | CacheDir() string 50 | // Get retrieves the given URL from the cache, or the remote server if it 51 | // isn't already present locally. 52 | Get(url string) (io.ReadCloser, error) 53 | // GetIfOlder retrieves the given URL from the cache, or the remote server 54 | // if it isn't already present locally OR the provided duration since now 55 | // has expired. 56 | GetIfOlder(url string, dur time.Duration) (io.ReadCloser, error) 57 | // Clean will wipe the contents of the cache. 58 | Clean() error 59 | } 60 | 61 | // New creates a new HTTPCache using the given directory 62 | func New(dir string) HTTPCache { 63 | return &httpCache{ 64 | cacheDir: dir, 65 | get: http.Get, 66 | } 67 | } 68 | 69 | type httpCache struct { 70 | cacheDir string 71 | get func(url string) (*http.Response, error) 72 | } 73 | 74 | func (h *httpCache) CacheDir() string { return h.cacheDir } 75 | 76 | func (h *httpCache) Clean() error { 77 | if h.cacheDir == "" { 78 | return errors.New("No cache directory detected") 79 | } 80 | log.Info("Wiping cache directory:", h.cacheDir) 81 | return os.RemoveAll(h.cacheDir) 82 | } 83 | 84 | func (h *httpCache) cachePathForURL(url string) (string, error) { 85 | cacheName, err := util.CalculateSHA256Sum(strings.NewReader(url)) 86 | if err != nil { 87 | return "", err 88 | } 89 | return path.Join(h.cacheDir, cacheName), nil 90 | } 91 | 92 | func (h *httpCache) GetIfOlder(url string, dur time.Duration) (io.ReadCloser, error) { 93 | // If cache is setup check it first 94 | if !NoCache && h.cacheDir != "" { 95 | log.Debugf("Checking local cache for the contents of %q\n", url) 96 | cachePath, err := h.cachePathForURL(url) 97 | if err != nil { 98 | return nil, err 99 | } 100 | if fileExists(cachePath) { 101 | if dur > 0 { 102 | stat, err := os.Stat(cachePath) 103 | if err != nil { 104 | return nil, err 105 | } 106 | if stat.ModTime().Add(dur).After(time.Now()) { 107 | log.Debugf("Serving request from local cached item %q\n", cachePath) 108 | return os.Open(cachePath) 109 | } 110 | log.Debugf("Cached item for %q is older than %v\n", url, dur) 111 | } else { 112 | log.Debugf("Serving request from local cached item %q\n", cachePath) 113 | return os.Open(cachePath) 114 | } 115 | } 116 | } 117 | 118 | // We need to download the file 119 | log.Debug("Performing HTTP GET to", url) 120 | resp, err := h.get(url) 121 | if err != nil { 122 | return nil, err 123 | } 124 | if resp.StatusCode != http.StatusOK { 125 | defer resp.Body.Close() 126 | body, err := ioutil.ReadAll(resp.Body) 127 | if err != nil { 128 | return nil, err 129 | } 130 | return nil, fmt.Errorf("error retrieving %q: %s", url, string(body)) 131 | } 132 | 133 | // If the cache is not configured, return the raw response body 134 | // so it can be closed properly by the caller. 135 | if NoCache || h.cacheDir == "" { 136 | log.Debug("Caching is not enabled, returning raw response object") 137 | return resp.Body, nil 138 | } 139 | 140 | // We have a local cache, save the object for future use 141 | defer resp.Body.Close() 142 | log.Debug("Calculating filename for new cache object") 143 | cachePath, err := h.cachePathForURL(url) 144 | if err != nil { 145 | return nil, err 146 | } 147 | log.Debugf("Writing %q to %q\n", url, cachePath) 148 | f, err := os.OpenFile(cachePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) 149 | if err != nil { 150 | return nil, err 151 | } 152 | if _, err := io.Copy(f, resp.Body); err != nil { 153 | return nil, err 154 | } 155 | if err := f.Close(); err != nil { 156 | return nil, err 157 | } 158 | 159 | return os.Open(cachePath) 160 | } 161 | 162 | func (h *httpCache) Get(url string) (io.ReadCloser, error) { 163 | return h.GetIfOlder(url, -1) 164 | } 165 | 166 | func fileExists(path string) bool { 167 | info, err := os.Stat(path) 168 | if os.IsNotExist(err) { 169 | return false 170 | } 171 | if err != nil { 172 | log.Error("Unexpected error from os.Stat:", err) 173 | return false 174 | } 175 | return !info.IsDir() 176 | } 177 | -------------------------------------------------------------------------------- /pkg/cmd/nodes.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os" 7 | "os/user" 8 | "path" 9 | "syscall" 10 | 11 | "github.com/spf13/cobra" 12 | "golang.org/x/crypto/ssh/terminal" 13 | 14 | "github.com/tinyzimmer/k3p/pkg/cluster" 15 | "github.com/tinyzimmer/k3p/pkg/cluster/node" 16 | "github.com/tinyzimmer/k3p/pkg/log" 17 | "github.com/tinyzimmer/k3p/pkg/types" 18 | ) 19 | 20 | var ( 21 | nodeAddRole string 22 | nodeRemoteLeader string 23 | nodeConnectOpts *types.NodeConnectOptions 24 | nodeAddOpts *types.AddNodeOptions 25 | nodeRemoveOpts *types.RemoveNodeOptions 26 | ) 27 | 28 | func init() { 29 | var currentUser *user.User 30 | var err error 31 | if currentUser, err = user.Current(); err != nil { 32 | log.Fatal(err) 33 | } 34 | 35 | nodeConnectOpts = &types.NodeConnectOptions{} 36 | nodeAddOpts = &types.AddNodeOptions{} 37 | nodeRemoveOpts = &types.RemoveNodeOptions{} 38 | 39 | var defaultKeyArg string 40 | defaultKeyPath := path.Join(currentUser.HomeDir, ".ssh", "id_rsa") 41 | if _, err := os.Stat(defaultKeyPath); err == nil { 42 | defaultKeyArg = defaultKeyPath 43 | } 44 | 45 | nodesCmd.PersistentFlags().StringVarP(&nodeConnectOpts.SSHUser, "ssh-user", "u", currentUser.Username, "The remote user to use for SSH authentication") 46 | nodesCmd.PersistentFlags().StringVarP(&nodeConnectOpts.SSHKeyFile, "private-key", "k", defaultKeyArg, "A private key to use for SSH authentication, if not provided you will be prompted for a password") 47 | nodesCmd.PersistentFlags().IntVarP(&nodeConnectOpts.SSHPort, "ssh-port", "p", 22, "The port to use when connecting to the remote instance over SSH") 48 | nodesCmd.PersistentFlags().StringVarP(&nodeRemoteLeader, "leader", "L", "", `The IP address or DNS name of the leader of the cluster. 49 | 50 | When left unset, the machine running k3p is assumed to be the leader of the cluster. Otherwise, 51 | the provided host is remoted into, with the same connection options as for the new node in case 52 | of an add, to retrieve the installation manifest. 53 | `) 54 | 55 | nodesAddCmd.Flags().StringVarP(&nodeAddRole, "node-role", "r", string(types.K3sRoleAgent), "Whether to join the instance as a 'server' or 'agent'") 56 | nodesAddCmd.RegisterFlagCompletionFunc("node-role", completeStringOpts([]string{"server", "agent"})) 57 | 58 | nodesRemoveCmd.Flags().BoolVar(&nodeRemoveOpts.Uninstall, "uninstall", false, "After the node is removed from the cluster, remote in and uninstall k3s") 59 | 60 | nodesCmd.AddCommand(nodesAddCmd) 61 | nodesCmd.AddCommand(nodesRemoveCmd) 62 | 63 | rootCmd.AddCommand(nodesCmd) 64 | } 65 | 66 | var nodesCmd = &cobra.Command{ 67 | Use: "node", 68 | Short: "Node management commands", 69 | } 70 | 71 | var nodesAddCmd = &cobra.Command{ 72 | Use: "add NODE [flags]", 73 | Short: "Add a new node to the cluster", 74 | Args: cobra.ExactArgs(1), 75 | RunE: addNode, 76 | } 77 | 78 | var nodesRemoveCmd = &cobra.Command{ 79 | Use: "remove NODE [flags]", 80 | Short: "Remove a node from the cluster by name or IP", 81 | Args: cobra.ExactArgs(1), 82 | RunE: removeNode, 83 | } 84 | 85 | func addNode(cmd *cobra.Command, args []string) error { 86 | nodeAddOpts.NodeConnectOptions = nodeConnectOpts 87 | nodeAddOpts.Address = args[0] 88 | 89 | switch types.K3sRole(nodeAddRole) { 90 | case types.K3sRoleServer: 91 | nodeAddOpts.NodeRole = types.K3sRoleServer 92 | case types.K3sRoleAgent: 93 | nodeAddOpts.NodeRole = types.K3sRoleAgent 94 | default: 95 | return fmt.Errorf("%q is not a valid node role", nodeAddRole) 96 | } 97 | 98 | if nodeAddOpts.SSHKeyFile == "" { 99 | fmt.Printf("Enter SSH Password for %s: ", nodeAddOpts.SSHUser) 100 | bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) 101 | if err != nil { 102 | return err 103 | } 104 | nodeAddOpts.SSHPassword = string(bytePassword) 105 | } 106 | 107 | leader, err := getLeader(nodeAddOpts.Address) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | log.Infof("Connecting to %s:%d\n", nodeAddOpts.Address, nodeAddOpts.SSHPort) 113 | newNode, err := node.Connect(nodeAddOpts.NodeConnectOptions) 114 | if err != nil { 115 | return err 116 | } 117 | 118 | return cluster.New(leader).AddNode(newNode, nodeAddOpts) 119 | } 120 | 121 | func removeNode(cmd *cobra.Command, args []string) error { 122 | if nodeRemoteLeader != "" && nodeConnectOpts.SSHKeyFile == "" { 123 | fmt.Printf("Enter SSH Password for %s: ", nodeAddOpts.SSHUser) 124 | bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) 125 | if err != nil { 126 | return err 127 | } 128 | nodeConnectOpts.SSHPassword = string(bytePassword) 129 | } 130 | nodeRemoveOpts.NodeConnectOptions = nodeConnectOpts 131 | 132 | target := args[0] 133 | if ip := net.ParseIP(target); ip != nil { 134 | // Valid IP address 135 | nodeRemoveOpts.IPAddress = target 136 | } else { 137 | // Assume it's a node name 138 | nodeRemoveOpts.Name = target 139 | } 140 | 141 | leader, err := getLeader(target) 142 | if err != nil { 143 | return err 144 | } 145 | 146 | return cluster.New(leader).RemoveNode(nodeRemoveOpts) 147 | } 148 | 149 | func getLeader(nodeName string) (types.Node, error) { 150 | if nodeRemoteLeader != "" { 151 | connectOpts := *nodeConnectOpts 152 | connectOpts.Address = nodeRemoteLeader 153 | log.Infof("Connecting to %s:%d\n", connectOpts.Address, connectOpts.SSHPort) 154 | return node.Connect(&connectOpts) 155 | } 156 | return node.Local(), nil 157 | } 158 | -------------------------------------------------------------------------------- /pkg/images/build_registry.go: -------------------------------------------------------------------------------- 1 | package images 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "time" 9 | 10 | dockertypes "github.com/docker/docker/api/types" 11 | "github.com/docker/docker/api/types/container" 12 | 13 | "github.com/tinyzimmer/k3p/pkg/images/registry" 14 | "github.com/tinyzimmer/k3p/pkg/log" 15 | "github.com/tinyzimmer/k3p/pkg/types" 16 | ) 17 | 18 | var requiredRegistryImages = []string{"registry:2", "busybox", registry.KubenabImage} 19 | 20 | func setOptDefaults(opts *types.BuildRegistryOptions) *types.BuildRegistryOptions { 21 | if opts.AppVersion == "" { 22 | opts.AppVersion = types.VersionLatest 23 | } 24 | 25 | if opts.PullPolicy == "" { 26 | opts.PullPolicy = types.PullPolicyAlways 27 | } 28 | 29 | return opts 30 | } 31 | 32 | func (d *dockerImageDownloader) BuildRegistry(opts *types.BuildRegistryOptions) (io.ReadCloser, error) { 33 | opts = setOptDefaults(opts) 34 | 35 | cli, err := getDockerClient() 36 | if err != nil { 37 | return nil, err 38 | } 39 | defer cli.Close() 40 | 41 | // Ensure all needed images are present 42 | userImages := sanitizeImageNameSlice(opts.Images) 43 | for _, img := range append(requiredRegistryImages, userImages...) { 44 | if err := ensureImagePulled(cli, img, opts.Arch, opts.PullPolicy); err != nil { 45 | return nil, err 46 | } 47 | } 48 | 49 | log.Info("Starting local private image registry") 50 | registryContainerConfig, registryHostConfig := registryContainerConfigs() 51 | log.Debugf("Registry container config: %+v\n", registryContainerConfig) 52 | log.Debugf("Registry host config: %+v\n", registryHostConfig) 53 | registryID, err := createAndStartContainer(cli, registryContainerConfig, registryHostConfig) 54 | if err != nil { 55 | return nil, err 56 | } 57 | defer func() { 58 | if err := cli.ContainerRemove(context.TODO(), registryID, dockertypes.ContainerRemoveOptions{ 59 | Force: true, 60 | RemoveVolumes: true, 61 | }); err != nil { 62 | log.Warning("Error removing registry container:", err) 63 | } 64 | }() 65 | 66 | // Fetch the host port that was bound to the registry 67 | log.Debugf("Inspecting container %s for exposed registry port\n", registryID) 68 | localPort, err := getHostPortForContainer(cli, registryID, "5000/tcp") 69 | if err != nil { 70 | return nil, err 71 | } 72 | log.Debugf("Local private registry is exposed on port %s\n", localPort) 73 | if err := waitForLocalRegistry(localPort, time.Second*10); err != nil { 74 | return nil, err 75 | } 76 | 77 | // Proxy user images into the registry 78 | for _, image := range userImages { 79 | log.Infof("Pushing %s to private registry\n", image) 80 | localImageName := fmt.Sprintf("localhost:%s/%s", localPort, image) 81 | log.Debug("Using local image name", localImageName) 82 | if err := cli.ImageTag(context.TODO(), image, localImageName); err != nil { 83 | return nil, err 84 | } 85 | rdr, err := cli.ImagePush(context.TODO(), localImageName, dockertypes.ImagePushOptions{ 86 | All: true, 87 | RegistryAuth: "fake", // https://github.com/moby/moby/issues/10983 88 | }) 89 | if err != nil { 90 | return nil, err 91 | } 92 | log.LevelReader(log.LevelDebug, rdr) 93 | } 94 | 95 | // Mount registry volumes into busybox image to take backup and commit the contents 96 | // to an image that can be used as an init container. 97 | log.Info("Exporting private registry contents to container image") 98 | busyboxConfig, busyboxHostConfig := registryVolumeContainerConfigs(registryID) 99 | log.Debugf("Busybox container config: %+v\n", busyboxConfig) 100 | log.Debugf("Busybox host config: %+v\n", busyboxHostConfig) 101 | volContainerID, err := createAndStartContainer(cli, busyboxConfig, busyboxHostConfig) 102 | if err != nil { 103 | return nil, err 104 | } 105 | defer func() { 106 | if err := cli.ContainerRemove(context.TODO(), volContainerID, dockertypes.ContainerRemoveOptions{ 107 | Force: true, 108 | RemoveVolumes: true, 109 | }); err != nil { 110 | log.Warning("Error removing registry volume container:", err) 111 | } 112 | }() 113 | // Wait for the tar process to finish 114 | log.Debug("Waiting for container process to finish") 115 | ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) // make configurable 116 | defer cancel() 117 | statusCh, errCh := cli.ContainerWait(ctx, volContainerID, container.WaitConditionNotRunning) 118 | select { 119 | case err := <-errCh: 120 | if err != nil { 121 | return nil, err 122 | } 123 | case res := <-statusCh: 124 | logs, err := cli.ContainerLogs(context.TODO(), volContainerID, dockertypes.ContainerLogsOptions{ 125 | ShowStdout: true, ShowStderr: true, 126 | }) 127 | if err != nil { 128 | log.Debug("Failed to retrieve container logs for", volContainerID, ":", err) 129 | } else { 130 | defer logs.Close() 131 | log.Debug("Container logs for", volContainerID) 132 | log.LevelReader(log.LevelDebug, logs) 133 | } 134 | if res.StatusCode != 0 { 135 | return nil, errors.New("Registry data backup exited with non-zero status code") 136 | } 137 | } 138 | 139 | // Commit the registry volume container to an image 140 | _, err = cli.ContainerCommit(context.TODO(), volContainerID, dockertypes.ContainerCommitOptions{Reference: opts.RegistryImageName()}) 141 | if err != nil { 142 | return nil, err 143 | } 144 | 145 | // Save all images for the registry 146 | return cli.ImageSave(context.TODO(), append(requiredRegistryImages, opts.RegistryImageName())) 147 | } 148 | -------------------------------------------------------------------------------- /pkg/parser/helm.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "strings" 10 | "text/template" 11 | 12 | "github.com/Masterminds/sprig" 13 | "gopkg.in/yaml.v2" 14 | "helm.sh/helm/v3/pkg/chart/loader" 15 | "helm.sh/helm/v3/pkg/chartutil" 16 | "helm.sh/helm/v3/pkg/engine" 17 | 18 | "github.com/tinyzimmer/k3p/pkg/log" 19 | "github.com/tinyzimmer/k3p/pkg/types" 20 | "github.com/tinyzimmer/k3p/pkg/util" 21 | ) 22 | 23 | // TODO: Makes targetNamespace configurable for charts 24 | var helmCRTmpl = template.Must(template.New("helm-cr").Funcs(sprig.TxtFuncMap()).Parse(`apiVersion: helm.cattle.io/v1 25 | kind: HelmChart 26 | metadata: 27 | name: {{ .Name }} 28 | namespace: kube-system 29 | spec: 30 | targetNamespace: default 31 | chart: https://%{KUBERNETES_API}%/static/k3p/{{ .Filename }} 32 | {{- if .ValuesContent }} 33 | valuesContent: |- 34 | {{ .ValuesContent | nindent 4 }} 35 | {{- end }} 36 | `)) 37 | 38 | func isHelmArchive(file string) bool { 39 | log.Debug("Attempting to load", file, "as helm chart") 40 | _, err := loader.Load(file) 41 | return err == nil 42 | } 43 | 44 | func (p *ManifestParser) detectImagesFromHelmChart(chartPath string) ([]string, error) { 45 | images := make([]string, 0) 46 | 47 | chart, err := loader.Load(chartPath) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | var helmVals chartutil.Values 53 | if p.PackageConfig != nil { 54 | if vals, ok := p.PackageConfig.HelmValues[chart.Name()]; ok { 55 | raw, err := yaml.Marshal(vals) 56 | if err != nil { 57 | return nil, err 58 | } 59 | helmVals, err = chartutil.ReadValues(raw) 60 | if err != nil { 61 | return nil, err 62 | } 63 | log.Debugf("Using the following values for chart %q: %+v\n", chart.Name(), helmVals) 64 | } 65 | } 66 | 67 | if err := chartutil.ProcessDependencies(chart, helmVals); err != nil { 68 | return nil, err 69 | } 70 | 71 | options := chartutil.ReleaseOptions{ 72 | Name: "k3p-build", 73 | Namespace: "default", 74 | Revision: 1, 75 | IsInstall: true, 76 | IsUpgrade: false, 77 | } 78 | valuesToRender, err := chartutil.ToRenderValues(chart, helmVals, options, nil) 79 | if err != nil { 80 | return nil, err 81 | } 82 | if err != nil { 83 | return nil, err 84 | } 85 | 86 | log.Debugf("Rendering helm chart %q to kubernetes manifests\n", chart.Name()) 87 | objects, err := engine.Render(chart, valuesToRender) 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | // iterate all the yaml objects in the rendered templates 93 | for _, rendered := range objects { 94 | // Check if this is empty space 95 | if len(strings.TrimSpace(rendered)) == 0 { 96 | continue 97 | } 98 | // Decode the object 99 | obj, err := p.Decode([]byte(rendered)) 100 | if err != nil { 101 | log.Debugf("Skipping invalid kubernetes object in rendered helm template: %s\n", err.Error()) 102 | continue 103 | } 104 | // Append any images to the local images to be downloaded 105 | if objImgs := parseObjectForImages(obj); len(objImgs) > 0 { 106 | images = appendIfMissing(images, objImgs...) 107 | } 108 | } 109 | 110 | return images, nil 111 | } 112 | 113 | func (p *ManifestParser) packageHelmChartToArtifacts(chartPath string) ([]*types.Artifact, error) { 114 | chart, err := loader.Load(chartPath) 115 | if err != nil { 116 | return nil, err 117 | } 118 | 119 | var valuesContent string 120 | if p.PackageConfig != nil { 121 | rawValues, err := p.GetHelmValues(chart.Name()) 122 | if err == nil { 123 | valuesContent = string(rawValues) 124 | } else { 125 | log.Debugf("Could not load helm values for chart %s: %s\n", chart.Name(), err.Error()) 126 | } 127 | } 128 | 129 | // package the chart to a temp file 130 | var packagedChartBytes []byte 131 | var packagedChartFilename string 132 | if ok, err := chartutil.IsChartDir(chartPath); err == nil && ok { 133 | // Chart is a directory that needs to be packaged 134 | tmpDir, err := util.GetTempDir() 135 | if err != nil { 136 | return nil, err 137 | } 138 | defer os.RemoveAll(tmpDir) 139 | log.Debugf("Packaging helm chart %q to %q\n", chart.Name(), tmpDir) 140 | chartPkg, err := chartutil.Save(chart, tmpDir) 141 | if err != nil { 142 | return nil, err 143 | } 144 | log.Debugf("Produced chart package at %q\n", chartPkg) 145 | packagedChartFilename = path.Base(chartPkg) 146 | packagedChartBytes, err = ioutil.ReadFile(chartPkg) 147 | if err != nil { 148 | return nil, err 149 | } 150 | } else { 151 | // Chart is already packaged 152 | log.Debugf("Chart at %q is already packaged, adding directly to manifest\n", chartPath) 153 | packagedChartFilename = path.Base(chartPath) 154 | packagedChartBytes, err = ioutil.ReadFile(chartPath) 155 | if err != nil { 156 | return nil, err 157 | } 158 | } 159 | 160 | stripExt := strings.TrimSuffix(path.Base(chartPath), ".tgz") 161 | 162 | var out bytes.Buffer 163 | if err := helmCRTmpl.Execute(&out, map[string]string{ 164 | "Name": chart.Name(), 165 | "Filename": packagedChartFilename, 166 | "ValuesContent": valuesContent, 167 | }); err != nil { 168 | return nil, err 169 | } 170 | outBytes := out.Bytes() 171 | return []*types.Artifact{ 172 | { 173 | Type: types.ArtifactManifest, 174 | Name: fmt.Sprintf("%s-helm-chart.yaml", stripExt), 175 | Body: ioutil.NopCloser(bytes.NewReader(outBytes)), 176 | Size: int64(len(outBytes)), 177 | }, 178 | { 179 | Type: types.ArtifactStatic, 180 | Name: packagedChartFilename, 181 | Body: ioutil.NopCloser(bytes.NewReader(packagedChartBytes)), 182 | Size: int64(len(packagedChartBytes)), 183 | }, 184 | }, nil 185 | } 186 | -------------------------------------------------------------------------------- /examples/ha/README.md: -------------------------------------------------------------------------------- 1 | ## HA Deployments 2 | 3 | In terms of the contents of the packag we again use a simple `whoami` example, except with this time specifying pod anti-affinity to ensure pods are not co-located on the same node. 4 | K3p can be used to add new nodes to the cluster either locally via the `install` command, or remotely via the `node add` command. 5 | 6 | ### Creating the Initial Node 7 | 8 | With the experimental k3s embedded etcd HA, one node has to be started with the `--cluster-init` flag, and then additional control-plane instances can be added through joining the initial node. 9 | 10 | With the package in this directory already built, SSH in to your first host and run `k3p install` with the `--init-ha` flag. (This can also be done remotely with the `--host` flag). 11 | 12 | ```bash 13 | [core@coreos1 ~]$ sudo k3p install package.tar --init-ha 14 | 15 | 2020/12/14 12:52:47 [INFO] Loading the archive 16 | 2020/12/14 12:52:48 [INFO] Copying the archive to the rancher installation directory 17 | 2020/12/14 12:52:49 [INFO] Generating a node token for additional control-plane instances 18 | 2020/12/14 12:52:49 [INFO] Installing binaries to /usr/local/bin 19 | 2020/12/14 12:52:49 [INFO] Installing scripts to /usr/local/bin/k3p-scripts 20 | 2020/12/14 12:52:49 [INFO] Installing images to /var/lib/rancher/k3s/agent/images 21 | 2020/12/14 12:52:50 [INFO] Installing manifests to /var/lib/rancher/k3s/server/manifests 22 | 2020/12/14 12:52:50 [INFO] Running k3s installation script 23 | 2020/12/14 12:52:50 [K3S] [INFO] Skipping k3s download and verify 24 | 2020/12/14 12:52:50 [K3S] [INFO] Skipping installation of SELinux RPM 25 | 2020/12/14 12:52:50 [K3S] [INFO] Creating /usr/local/bin/kubectl symlink to k3s 26 | 2020/12/14 12:52:50 [K3S] [INFO] Creating /usr/local/bin/crictl symlink to k3s 27 | 2020/12/14 12:52:50 [K3S] [INFO] Skipping /usr/local/bin/ctr symlink to k3s, command exists in PATH at /usr/bin/ctr 28 | 2020/12/14 12:52:50 [K3S] [INFO] Creating killall script /usr/local/bin/k3s-killall.sh 29 | 2020/12/14 12:52:50 [K3S] [INFO] Creating uninstall script /usr/local/bin/k3s-uninstall.sh 30 | 2020/12/14 12:52:50 [K3S] [INFO] env: Creating environment file /etc/systemd/system/k3s.service.env 31 | 2020/12/14 12:52:50 [K3S] [INFO] systemd: Creating service file /etc/systemd/system/k3s.service 32 | 2020/12/14 12:52:50 [K3S] [INFO] systemd: Enabling k3s unit 33 | 2020/12/14 12:52:50 [K3S] Created symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service. 34 | 2020/12/14 12:52:51 [K3S] [INFO] systemd: Starting k3s 35 | 2020/12/14 12:52:59 [INFO] The cluster has been installed 36 | 2020/12/14 12:52:59 [INFO] You can view the cluster by running `k3s kubectl cluster-info` 37 | 38 | # A token was generated for joining new control-plane instances during the install. 39 | # A pre-generated one can also be used. To retrieve the generated one you can run 40 | [core@coreos1 ~]$ sudo k3p token get server 41 | fFUiC96GBQ69XgENdvhabseBd53vSUVWuYhrLJKVRX08a3M9RA8qSYypBxLMX0iCEPBnWl6BmZ6WKIw4pAhtbQYhMWveiGI3YbkGMkwJQnTfuTnkBzzMIvsitvBiwqg3 42 | 43 | # So far we have a single node, and we are unable to schedule two of our pods 44 | [core@coreos1 ~]$ sudo k3s kubectl get node 45 | NAME STATUS ROLES AGE VERSION 46 | coreos1 Ready etcd,master 68s v1.19.4+k3s1 47 | 48 | [core@coreos1 ~]$ sudo k3s kubectl get pod 49 | NAME READY STATUS RESTARTS AGE 50 | whoami-5f47859667-87l9q 1/1 Running 0 63s 51 | whoami-5f47859667-l77k6 0/1 Pending 0 63s 52 | whoami-5f47859667-n8jvh 0/1 Pending 0 63s 53 | ``` 54 | 55 | To join a second and third instance to the cluster there are two (actually three) ways we can do this. The first way is to install the package again to the other instances, using the `--join` flag to signal joining an existing cluster. 56 | 57 | ```bash 58 | [core@coreos2 ~]$ sudo k3p install package.tar \ 59 | --join https://172.18.64.84:6443 \ # The IP and API port of the first instance 60 | --join-role server \ # Join as a server instance (the default option is as an agent and uses a different token) 61 | --join-token fFUiC96GBQ69XgENdvhabseBd53vSUVWuYhrLJKVRX08a3M9RA8qSYypBxLMX0iCEPBnWl6BmZ6WKIw4pAhtbQYhMWveiGI3YbkGMkwJQnTfuTnkBzzMIvsitvBiwqg3 62 | 63 | # ... 64 | # ... 65 | ``` 66 | 67 | You can also use `k3p node add` from the initial node to bring in new instances using SSH. If you have public key authentication setup you can use that, otherwise it will prompt for a password. 68 | 69 | You can also do this from a remote instance with the `--leader` flag assuming it uses the same SSH credentials as the new node you are adding. 70 | 71 | ```bash 72 | [core@coreos1 ~]$ sudo k3p node add 172.18.64.91 \ # The remote address of the node 73 | --ssh-user core \ # The user to use for SSH 74 | --private-key ~/.ssh/id_rsa \ # The SSH private key (or omit to be prompted for a password) 75 | --node-role server # Join as a server 76 | 77 | # ... 78 | # ... 79 | ``` 80 | 81 | Once that is done you will have a highly available cluster and deployment 82 | 83 | ```bash 84 | [core@coreos1 ~]$ sudo k3s kubectl get node 85 | NAME STATUS ROLES AGE VERSION 86 | coreos1 Ready etcd,master 11m v1.19.4+k3s1 87 | coreos2 Ready etcd,master 6m20s v1.19.4+k3s1 88 | coreos3 Ready etcd,master 2m57s v1.19.4+k3s1 89 | 90 | [core@coreos1 ~]$ sudo k3s kubectl get pod 91 | NAME READY STATUS RESTARTS AGE 92 | whoami-5f47859667-87l9q 1/1 Running 0 11m 93 | whoami-5f47859667-l77k6 1/1 Running 0 11m 94 | whoami-5f47859667-n8jvh 1/1 Running 0 11m 95 | ``` -------------------------------------------------------------------------------- /doc/k3p_install.md: -------------------------------------------------------------------------------- 1 | ## k3p install 2 | 3 | Install the given package to the system 4 | 5 | ### Synopsis 6 | 7 | 8 | The install command can be used to distribute a package built with "k3p build". 9 | 10 | The command takes a single argument (with optional flags) of the filesystem path or web URL 11 | where the package resides. Additional flags provide the ability to initialize clustering (HA), 12 | join existing servers, or pass custom arguments to the k3s agent/server processes. 13 | 14 | Example 15 | 16 | $> k3p install /path/on/filesystem.tar 17 | $> k3p install https://example.com/package.tar 18 | 19 | When running on the local system like above, you will need to have root privileges. You can also 20 | direct the installation at a remote system over SSH via the --host flag. This will require the 21 | remote user having passwordless sudo available to them. 22 | 23 | $> k3p install package.tar --host 192.168.1.100 [SSH_FLAGS] 24 | 25 | See the help below for additional information on available flags. 26 | 27 | 28 | ``` 29 | k3p install PACKAGE [flags] 30 | ``` 31 | 32 | ### Options 33 | 34 | ``` 35 | --accept-defaults Accept the defaults for any package configurations, default behavior is to prompt for all unprovided values 36 | --accept-eula Automatically accept any EULA included with the package 37 | --agents int DOCKER ONLY: The number of agents to run in the cluster 38 | --api-port int The port for the k3s server to bind to (default 6443) 39 | --cluster-name string DOCKER ONLY: Override the name of the cluster (defaults to the package name) 40 | -D, --docker Install the package to a docker container on the local system. 41 | -h, --help help for install 42 | -H, --host string The IP or DNS name of a remote host to perform the installation against 43 | --init-ha When set, this server will run with the --cluster-init flag to enable clustering, 44 | and a token will be generated for adding additional servers to the cluster with 45 | "--join-role server". You may optionally use the --join-token flag to provide a 46 | pre-generated one. 47 | -j, --join string When installing an agent instance, the address of the server to join (e.g. https://myserver:6443) 48 | -r, --join-role string Specify whether to join the cluster as a "server" or "agent" (default "agent") 49 | -t, --join-token string When installing an additional agent or server instance, the node token to use. 50 | 51 | For new agents, this can be retrieved with "k3p token get agent" or in 52 | "/var/lib/rancher/k3s/server/node-token" on any of the server instances. 53 | For new servers, this value was either provided to or generated by 54 | "k3s install --init-ha" and can be retrieved from that server with 55 | "k3p token get server". When used with --init-ha, the provided token will 56 | be used for registering new servers, instead of one being generated. 57 | --k3s-agent-arg stringArray Extra arguments to pass to the k3s agent process, for more details see: 58 | https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config 59 | 60 | --k3s-server-arg stringArray Extra arguments to pass to the k3s server process, for more details see: 61 | https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config 62 | 63 | --kubeconfig-mode string The mode to set on the k3s kubeconfig. Default is to only allow root access 64 | -n, --node-name string An optional name to give this node in the cluster 65 | -k, --private-key string The path to a private key to use when authenticating against the remote host, 66 | if not provided you will be prompted for a password (default "/home//.ssh/id_rsa") 67 | -p, --publish stringArray DOCKER ONLY: Additional port mappings in the same format as used for k3d 68 | --resolv-conf string The path of a resolv-conf file to use when configuring DNS in the cluster. 69 | When used with the --host flag, the path must reside on the remote system (this will change in the future). 70 | --servers int DOCKER ONLY: The number of servers to run in the cluster (default 1) 71 | --set stringArray Values to set to configurations in the package in the format of --set = 72 | -P, --ssh-port int The port to use when connecting to the remote host over SSH (default 22) 73 | -u, --ssh-user string The username to use when authenticating against the remote host (default "") 74 | -f, --values string An optional json or yaml file containing key-value pairs of package configurations 75 | --write-kubeconfig string Write a copy of the admin client to this file 76 | ``` 77 | 78 | ### Options inherited from parent commands 79 | 80 | ``` 81 | --cache-dir string Override the default location for cached k3s assets (default "/home//.k3p/cache") 82 | --tmp-dir string Override the default tmp directory (default "/tmp") 83 | -v, --verbose Enable verbose logging 84 | ``` 85 | 86 | ### SEE ALSO 87 | 88 | * [k3p](k3p.md) - k3p is a k3s packaging and delivery utility 89 | 90 | -------------------------------------------------------------------------------- /pkg/cmd/build.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "runtime" 10 | "strings" 11 | "time" 12 | 13 | "github.com/spf13/cobra" 14 | 15 | "github.com/tinyzimmer/k3p/pkg/build" 16 | "github.com/tinyzimmer/k3p/pkg/cache" 17 | "github.com/tinyzimmer/k3p/pkg/log" 18 | "github.com/tinyzimmer/k3p/pkg/types" 19 | ) 20 | 21 | var ( 22 | buildPullPolicy string 23 | buildOpts *types.BuildOptions 24 | ) 25 | 26 | func init() { 27 | cwd, err := os.Getwd() 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | 32 | var defaultConfig string 33 | if _, err := os.Stat(path.Join(cwd, "k3p.yaml")); err == nil { 34 | defaultConfig = path.Join(cwd, "k3p.yaml") 35 | } 36 | 37 | buildOpts = &types.BuildOptions{} 38 | 39 | buildCmd.Flags().StringVarP(&buildOpts.Name, "name", "n", "", `The name to give the package, if not provided one will be generated`) 40 | buildCmd.Flags().StringVarP(&buildOpts.BuildVersion, "version", "V", types.VersionLatest, "The version to tag the package") 41 | buildCmd.Flags().StringVar(&buildOpts.K3sVersion, "k3s-version", types.VersionLatest, "A specific k3s version to bundle with the package, overrides --channel") 42 | buildCmd.Flags().StringVarP(&buildOpts.K3sChannel, "channel", "C", "stable", "The release channel to retrieve the version of k3s from") 43 | buildCmd.Flags().StringArrayVarP(&buildOpts.ManifestDirs, "manifests", "m", []string{cwd}, "Directories to scan for kubernetes manifests and charts, defaults to the current directory, can be specified multiple times") 44 | buildCmd.Flags().StringSliceVarP(&buildOpts.Excludes, "exclude", "e", []string{}, "Directories to exclude when reading the manifest directory") 45 | buildCmd.Flags().StringVarP(&buildOpts.Arch, "arch", "a", runtime.GOARCH, "The architecture to package the distribution for. Only (amd64, arm, and arm64 are supported)") 46 | buildCmd.Flags().StringVarP(&buildOpts.ImageFile, "image-file", "I", "", "A file containing a list of extra images to bundle with the archive") 47 | buildCmd.Flags().StringSliceVarP(&buildOpts.Images, "images", "i", []string{}, "A comma separated list of images to include with the archive") 48 | buildCmd.Flags().StringVarP(&buildOpts.EULAFile, "eula", "E", "", "A file containing an End User License Agreement to display to the user upon installing the package") 49 | buildCmd.Flags().StringVarP(&buildOpts.Output, "output", "o", path.Join(cwd, "package.tar"), "The file to save the distribution package to") 50 | buildCmd.Flags().BoolVar(&buildOpts.ExcludeImages, "exclude-images", false, "Don't include container images with the final archive") 51 | buildCmd.Flags().StringVar(&buildPullPolicy, "pull-policy", string(types.PullPolicyAlways), "The pull policy to use when bundling container images (valid options always,never,ifnotpresent [case-insensitive])") 52 | buildCmd.Flags().StringVarP(&buildOpts.ConfigFile, "config", "c", defaultConfig, "An optional file providing variables and other configurations to be used at installation, if a k3p.yaml in the current directory exists it will be used automatically") 53 | buildCmd.Flags().BoolVarP(&cache.NoCache, "no-cache", "N", false, "Disable the use of the local cache when downloading assets") 54 | buildCmd.Flags().BoolVar(&buildOpts.Compress, "compress", false, "Whether to apply zst encryption to the package, it will usually require the same k3p release to decompress.") 55 | buildCmd.Flags().BoolVar(&buildOpts.RunFile, "run-file", false, "Whether to bundle the final archive into a self-installing run file") 56 | buildCmd.Flags().BoolVar(&buildOpts.CreateRegistry, "build-registry", false, "Bundle container images into a private registry instead of just raw tar balls") 57 | 58 | buildCmd.MarkFlagDirname("exclude") 59 | buildCmd.MarkFlagDirname("manifests") 60 | buildCmd.MarkFlagFilename("config", "json", "yaml", "yml") 61 | buildCmd.RegisterFlagCompletionFunc("pull-policy", completeStringOpts([]string{string(types.PullPolicyAlways), string(types.PullPolicyIfNotPresent), string(types.PullPolicyNever)})) 62 | buildCmd.RegisterFlagCompletionFunc("arch", completeStringOpts([]string{"amd64", "arm64", "arm"})) 63 | buildCmd.RegisterFlagCompletionFunc("channel", completeChannels) 64 | 65 | rootCmd.AddCommand(buildCmd) 66 | } 67 | 68 | var buildCmd = &cobra.Command{ 69 | Use: "build", 70 | Short: "Build a k3s distribution package", 71 | PreRunE: func(cmd *cobra.Command, args []string) error { 72 | // validate pull policy first 73 | switch types.PullPolicy(strings.ToLower(buildPullPolicy)) { 74 | case types.PullPolicyAlways: 75 | buildOpts.PullPolicy = types.PullPolicyAlways 76 | case types.PullPolicyNever: 77 | buildOpts.PullPolicy = types.PullPolicyNever 78 | case types.PullPolicyIfNotPresent: 79 | buildOpts.PullPolicy = types.PullPolicyIfNotPresent 80 | default: 81 | return fmt.Errorf("%s is not a valid pull policy", buildPullPolicy) 82 | } 83 | return nil 84 | }, 85 | RunE: func(cmd *cobra.Command, args []string) error { 86 | builder, err := build.NewBuilder() 87 | if err != nil { 88 | return err 89 | } 90 | return builder.Build(buildOpts) 91 | }, 92 | } 93 | 94 | type channelResponse struct { 95 | Data []channel `json:"data"` 96 | } 97 | 98 | type channel struct { 99 | ID string `json:"id"` 100 | } 101 | 102 | func completeChannels(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { 103 | log.Verbose = false 104 | var res channelResponse 105 | resp, err := cache.DefaultCache.GetIfOlder("https://update.k3s.io/v1-release/channels", time.Hour*24) 106 | if err != nil { 107 | return nil, cobra.ShellCompDirectiveError 108 | } 109 | defer resp.Close() 110 | body, err := ioutil.ReadAll(resp) 111 | if err != nil { 112 | return nil, cobra.ShellCompDirectiveError 113 | } 114 | if err := json.Unmarshal(body, &res); err != nil { 115 | return nil, cobra.ShellCompDirectiveError 116 | } 117 | out := make([]string, len(res.Data)) 118 | for i, channel := range res.Data { 119 | out[i] = channel.ID 120 | } 121 | return out, cobra.ShellCompDirectiveDefault 122 | } 123 | -------------------------------------------------------------------------------- /pkg/types/installer.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | "strings" 7 | ) 8 | 9 | // Installer is an interface for laying a package manifest down on a system 10 | // and setting up K3s. 11 | type Installer interface { 12 | Install(node Node, pkg Package, opts *InstallOptions) error 13 | } 14 | 15 | // InstallOptions are options to pass to an installation 16 | type InstallOptions struct { 17 | // An optional name to give the node 18 | NodeName string 19 | // Whether to skip viewing any EULA included in the package 20 | AcceptEULA bool 21 | // The URL to an already running k3s server to join as an agent 22 | ServerURL string 23 | // The node token from an already running k3s server 24 | NodeToken string 25 | // An optional resolv conf to use when configuring DNS 26 | ResolvConf string 27 | // Optionally override the default k3s kubeconfig mode (0600) 28 | // It is a string so it can be passed directly as an env var 29 | KubeconfigMode string 30 | // The port that the k3s API server should listen on 31 | APIListenPort int 32 | // Extra arguments to pass to the k3s server process that are not included 33 | // in the package. This includes arguments to the agent running on a server. 34 | K3sServerArgs []string 35 | // Extra arguments to pass to the k3s agent process that are not included 36 | // in the package. 37 | K3sAgentArgs []string 38 | // Whether to run with --cluster-init 39 | InitHA bool 40 | // Whether to run as a server or agent 41 | K3sRole K3sRole 42 | // Variables contain substitutions to perform on manifests before 43 | // installing them to the system. 44 | Variables map[string]string 45 | // The password to use for authentication to the registry, if this is blank one will 46 | // be generated. 47 | RegistrySecret string 48 | // The node port that the private registry will listen on when installed. Defaults to 49 | // 30100. 50 | RegistryNodePort int 51 | // The path to a TLS certificate to use for the private registry. If left unset a 52 | // self-signed certificate chain is generated. 53 | RegistryTLSCertFile string 54 | // The path to an unencrypted TLS private key to use for the private registry that matches 55 | // the leaf certificate provided to RegistryTLSBundle. A key is generated if not provided. 56 | RegistryTLSKeyFile string 57 | // The path to the CA bundle for the provided TLS certificate 58 | RegistryTLSCAFile string 59 | } 60 | 61 | // GetRegistryNodePort returns the node port to use for a private-registry. 62 | func (opts *InstallOptions) GetRegistryNodePort() int { 63 | if opts.RegistryNodePort != 0 { 64 | return opts.RegistryNodePort 65 | } 66 | return DefaultRegistryPort 67 | } 68 | 69 | // DeepCopy creates a copy of these installation options. 70 | func (opts *InstallOptions) DeepCopy() *InstallOptions { 71 | newOpts := &InstallOptions{ 72 | NodeName: opts.NodeName, 73 | AcceptEULA: opts.AcceptEULA, 74 | ServerURL: opts.ServerURL, 75 | NodeToken: opts.NodeToken, 76 | ResolvConf: opts.ResolvConf, 77 | KubeconfigMode: opts.KubeconfigMode, 78 | APIListenPort: opts.APIListenPort, 79 | K3sServerArgs: make([]string, len(opts.K3sServerArgs)), 80 | K3sAgentArgs: make([]string, len(opts.K3sAgentArgs)), 81 | InitHA: opts.InitHA, 82 | K3sRole: opts.K3sRole, 83 | Variables: make(map[string]string), 84 | RegistrySecret: opts.RegistrySecret, 85 | RegistryNodePort: opts.RegistryNodePort, 86 | } 87 | copy(newOpts.K3sServerArgs, opts.K3sServerArgs) 88 | copy(newOpts.K3sAgentArgs, opts.K3sAgentArgs) 89 | for k, v := range opts.Variables { 90 | newOpts.Variables[k] = v 91 | } 92 | return newOpts 93 | } 94 | 95 | // ToExecOpts converts these install options into execute options to pass to a 96 | // node. 97 | func (opts *InstallOptions) ToExecOpts(cfg *PackageConfig) *ExecuteOptions { 98 | env := map[string]string{ 99 | "INSTALL_K3S_SKIP_DOWNLOAD": "true", 100 | } 101 | 102 | if opts.NodeName != "" { 103 | env["K3S_NODE_NAME"] = opts.NodeName 104 | } 105 | 106 | if opts.ResolvConf != "" { 107 | env["K3S_RESOLV_CONF"] = opts.ResolvConf 108 | } 109 | 110 | if opts.KubeconfigMode != "" { 111 | env["K3S_KUBECONFIG_MODE"] = opts.KubeconfigMode 112 | } 113 | 114 | if opts.NodeToken != "" { 115 | env["K3S_TOKEN"] = opts.NodeToken 116 | } 117 | 118 | if opts.ServerURL != "" { 119 | env["K3S_URL"] = opts.ServerURL 120 | } 121 | 122 | var execFields []string 123 | switch opts.K3sRole { 124 | case K3sRoleServer, "": 125 | execFields = append([]string{string(K3sRoleServer)}, opts.K3sServerArgs...) 126 | case K3sRoleAgent: 127 | execFields = append([]string{string(K3sRoleAgent)}, opts.K3sAgentArgs...) 128 | } 129 | 130 | // Build out an exec string from the configuration 131 | if cfg != nil { 132 | switch opts.K3sRole { 133 | case K3sRoleServer, "": 134 | execFields = cfg.ServerArgs(execFields) 135 | case K3sRoleAgent: 136 | execFields = cfg.AgentArgs(execFields) 137 | } 138 | } 139 | 140 | if opts.APIListenPort != 0 && opts.K3sRole != K3sRoleAgent { 141 | execFields = append(execFields, fmt.Sprintf("--https-listen-port=%d", opts.APIListenPort)) 142 | } 143 | 144 | if args := strings.Join(execFields, " "); args != "" { 145 | env["INSTALL_K3S_EXEC"] = args 146 | } 147 | 148 | secrets := []string{} 149 | if opts.NodeToken != "" { 150 | secrets = []string{opts.NodeToken} 151 | } 152 | 153 | return &ExecuteOptions{ 154 | Env: env, 155 | Command: fmt.Sprintf("sh %q", path.Join(K3sScriptsDir, "install.sh")), 156 | Secrets: secrets, 157 | } 158 | } 159 | 160 | // InstallConfig represents the values that were collected at installation time. It is used 161 | // to serialize the configuration used to disk for future node-add/join operations. 162 | type InstallConfig struct { 163 | // Options passed at installation 164 | InstallOptions *InstallOptions 165 | } 166 | 167 | // DeepCopy creates a copy of this InstallConfig. 168 | func (i *InstallConfig) DeepCopy() *InstallConfig { 169 | return &InstallConfig{ 170 | InstallOptions: i.InstallOptions.DeepCopy(), 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /pkg/cluster/manager.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "strings" 9 | "time" 10 | 11 | v1 "github.com/tinyzimmer/k3p/pkg/build/package/v1" 12 | "github.com/tinyzimmer/k3p/pkg/cluster/kubernetes" 13 | "github.com/tinyzimmer/k3p/pkg/cluster/node" 14 | "github.com/tinyzimmer/k3p/pkg/log" 15 | "github.com/tinyzimmer/k3p/pkg/types" 16 | "github.com/tinyzimmer/k3p/pkg/util" 17 | ) 18 | 19 | // New returns a new ClusterManager instance. 20 | func New(leader types.Node) types.ClusterManager { return &manager{leader: leader} } 21 | 22 | type manager struct{ leader types.Node } 23 | 24 | func (m *manager) getKubeconfig() ([]byte, error) { 25 | f, err := m.leader.GetFile(types.K3sKubeconfig) 26 | if err != nil { 27 | return nil, err 28 | } 29 | defer f.Close() 30 | body, err := ioutil.ReadAll(f) 31 | if err != nil { 32 | return nil, err 33 | } 34 | addr, err := m.leader.GetK3sAddress() 35 | if err != nil { 36 | return nil, err 37 | } 38 | return []byte(strings.Replace(string(body), "127.0.0.1", addr, 1)), nil 39 | } 40 | 41 | func (m *manager) RemoveNode(opts *types.RemoveNodeOptions) error { 42 | log.Debug("Retrieve kubeconfig from leader") 43 | cfg, err := m.getKubeconfig() 44 | if err != nil { 45 | return err 46 | } 47 | cli, err := kubernetes.New(cfg) 48 | if err != nil { 49 | return err 50 | } 51 | var nodeName string 52 | if opts.Name != "" { 53 | nodeName = opts.Name 54 | } else if opts.IPAddress != "" { 55 | log.Debug("Looking up node by IP", opts.IPAddress) 56 | node, err := cli.GetNodeByIP(opts.IPAddress) 57 | if err != nil { 58 | return err 59 | } 60 | nodeName = node.GetName() 61 | } 62 | 63 | if opts.Uninstall { 64 | if opts.IPAddress == "" { 65 | ip, err := cli.GetIPByNodeName(nodeName) 66 | if err != nil { 67 | return err 68 | } 69 | opts.NodeConnectOptions.Address = ip 70 | } else { 71 | opts.NodeConnectOptions.Address = opts.IPAddress 72 | } 73 | } 74 | 75 | log.Info("Deleting node", nodeName) 76 | if err := cli.RemoveNode(nodeName); err != nil { 77 | return err 78 | } 79 | 80 | // Wait for the node to be deleted 81 | log.Infof("Waiting for %q to be removed from the cluster\n", nodeName) 82 | var failCount int 83 | ListNodesLoop: 84 | for { 85 | nodes, err := cli.ListNodes() 86 | if err != nil { 87 | if failCount > 3 { 88 | return err 89 | } 90 | log.Debug("Failure while listing nodes, retrying - error:", err.Error()) 91 | failCount++ 92 | continue ListNodesLoop 93 | } 94 | failCount = 0 95 | for _, node := range nodes { 96 | if node.GetName() == nodeName { 97 | log.Debug("Still waiting for node to be removed") 98 | time.Sleep(time.Second) 99 | continue ListNodesLoop 100 | } 101 | } 102 | break ListNodesLoop 103 | } 104 | 105 | if opts.Uninstall { 106 | log.Infof("Connecting to %s and uninstalling k3s\n", nodeName) 107 | oldNode, err := node.Connect(opts.NodeConnectOptions) 108 | if err != nil { 109 | return err 110 | } 111 | return oldNode.Execute(&types.ExecuteOptions{ 112 | Command: "k3s-uninstall.sh", // TODO: type cast somewhere 113 | }) 114 | } 115 | 116 | return nil 117 | } 118 | 119 | func (m *manager) AddNode(newNode types.Node, opts *types.AddNodeOptions) error { 120 | 121 | remoteAddr, err := m.leader.GetK3sAddress() 122 | if err != nil { 123 | return err 124 | } 125 | log.Debug("K3s is listening on", remoteAddr) 126 | 127 | // The reason we send the manifest over in pieces is because I was having strange bugs 128 | // with trying to send it over with the k3p binary and extract on the remote host. 129 | // 130 | // The tarball was moving over, but then ended up being an empty file on the other end. 131 | // Loading it locally and sending it in pieces works for now. 132 | log.Info("Loading package manifest") 133 | f, err := m.leader.GetFile(types.InstalledPackageFile) 134 | if err != nil { 135 | return err 136 | } 137 | defer f.Close() 138 | 139 | pkg, err := v1.Load(f) 140 | if err != nil { 141 | return err 142 | } 143 | defer pkg.Close() 144 | 145 | var tokenRdr io.ReadCloser 146 | switch opts.NodeRole { 147 | case types.K3sRoleServer: 148 | log.Debug("Reading server join token from", types.ServerTokenFile) 149 | tokenRdr, err = m.leader.GetFile(types.ServerTokenFile) 150 | case types.K3sRoleAgent: 151 | log.Debug("Reading agent join token from", types.AgentTokenFile) 152 | tokenRdr, err = m.leader.GetFile(types.AgentTokenFile) 153 | default: 154 | return fmt.Errorf("Invalid node role %s", opts.NodeRole) 155 | } 156 | if err != nil { 157 | return err 158 | } 159 | defer tokenRdr.Close() 160 | 161 | token, err := ioutil.ReadAll(tokenRdr) 162 | if err != nil { 163 | return err 164 | } 165 | tokenStr := strings.TrimSpace(string(token)) 166 | 167 | log.Debug("Loading installed package configuration") 168 | var installedConfig types.InstallConfig 169 | cfgFile, err := m.leader.GetFile(types.InstalledConfigFile) 170 | if err != nil { 171 | return err 172 | } 173 | defer cfgFile.Close() 174 | body, err := ioutil.ReadAll(cfgFile) 175 | if err != nil { 176 | return err 177 | } 178 | if err := json.Unmarshal(body, &installedConfig); err != nil { 179 | return err 180 | } 181 | 182 | if err := util.SyncPackageToNode(newNode, pkg, &installedConfig); err != nil { 183 | return err 184 | } 185 | 186 | log.Infof("Joining instance as a new %s\n", opts.NodeRole) 187 | execOpts, err := buildInstallOpts(pkg, &installedConfig, remoteAddr, tokenStr, opts.NodeRole) 188 | if err != nil { 189 | return err 190 | } 191 | return newNode.Execute(execOpts) 192 | } 193 | 194 | func buildInstallOpts(pkg types.Package, cfg *types.InstallConfig, remoteAddr, token string, nodeRole types.K3sRole) (*types.ExecuteOptions, error) { 195 | opts := cfg.DeepCopy().InstallOptions 196 | pkgConf := pkg.GetMeta().DeepCopy().Sanitize().GetPackageConfig() 197 | if pkgConf != nil { 198 | if err := pkgConf.ApplyVariables(opts.Variables); err != nil { 199 | return nil, err 200 | } 201 | } 202 | opts.ServerURL = fmt.Sprintf("https://%s:%d", remoteAddr, cfg.InstallOptions.APIListenPort) 203 | opts.NodeToken = token 204 | opts.K3sRole = nodeRole 205 | return opts.ToExecOpts(pkgConf), nil 206 | } 207 | -------------------------------------------------------------------------------- /pkg/build/k3s_components.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "path" 10 | "strings" 11 | 12 | "github.com/tinyzimmer/k3p/pkg/cache" 13 | "github.com/tinyzimmer/k3p/pkg/log" 14 | "github.com/tinyzimmer/k3p/pkg/types" 15 | "github.com/tinyzimmer/k3p/pkg/util" 16 | ) 17 | 18 | const ( 19 | k3sScriptURL = "https://get.k3s.io" 20 | k3sReleasesRootURL = "https://github.com/k3s-io/k3s/releases" 21 | k3sChannelsRoot = "https://update.k3s.io/v1-release/channels" 22 | ) 23 | 24 | func (b *builder) downloadCoreK3sComponents(opts *types.BuildOptions) error { 25 | log.Info("Fetching checksums...") 26 | if err := b.downloadK3sChecksums(opts.K3sVersion, opts.Arch); err != nil { 27 | return err 28 | } 29 | 30 | log.Info("Fetching k3s install script...") 31 | if err := b.downloadK3sInstallScript(); err != nil { 32 | return err 33 | } 34 | 35 | log.Info("Fetching k3s binary...") 36 | if err := b.downloadK3sBinary(opts.K3sVersion, opts.Arch); err != nil { 37 | return err 38 | } 39 | 40 | if !opts.ExcludeImages { 41 | log.Info("Fetching k3s airgap images...") 42 | if err := b.downloadK3sAirgapImages(opts.K3sVersion, opts.Arch); err != nil { 43 | return err 44 | } 45 | } else { 46 | log.Info("Skipping bundling k3s airgap images with the package") 47 | } 48 | 49 | log.Info("Validating checksums...") 50 | if err := b.validateCheckSums(opts); err != nil { 51 | return err 52 | } 53 | 54 | return nil 55 | } 56 | 57 | func (b *builder) downloadK3sChecksums(version, arch string) error { 58 | rdr, err := cache.DefaultCache.Get(getDownloadURL(version, getDownloadChecksumsName(arch))) 59 | if err != nil { 60 | return err 61 | } 62 | artifact, err := util.ArtifactFromReader(types.ArtifactType("misc"), "k3s-sha256sums.txt", rdr) 63 | if err != nil { 64 | return err 65 | } 66 | return b.writer.Put(artifact) 67 | } 68 | 69 | func (b *builder) downloadK3sInstallScript() error { 70 | rdr, err := cache.DefaultCache.Get(k3sScriptURL) 71 | if err != nil { 72 | return err 73 | } 74 | artifact, err := util.ArtifactFromReader(types.ArtifactScript, "install.sh", rdr) 75 | if err != nil { 76 | return err 77 | } 78 | return b.writer.Put(artifact) 79 | } 80 | 81 | func (b *builder) downloadK3sAirgapImages(version, arch string) error { 82 | rdr, err := cache.DefaultCache.Get(getDownloadURL(version, getDownloadAirgapImagesName(arch))) 83 | if err != nil { 84 | return err 85 | } 86 | artifact, err := util.ArtifactFromReader(types.ArtifactImages, "k3s-airgap-images.tar", rdr) 87 | if err != nil { 88 | return err 89 | } 90 | return b.writer.Put(artifact) 91 | } 92 | 93 | func (b *builder) downloadK3sBinary(version, arch string) error { 94 | rdr, err := cache.DefaultCache.Get(getDownloadURL(version, getDownloadK3sBinName(arch))) 95 | if err != nil { 96 | return err 97 | } 98 | artifact, err := util.ArtifactFromReader(types.ArtifactBin, "k3s", rdr) 99 | if err != nil { 100 | return err 101 | } 102 | return b.writer.Put(artifact) 103 | } 104 | 105 | func (b *builder) validateCheckSums(opts *types.BuildOptions) error { 106 | // Queue up extra check to make sure we visited each 107 | var binValid, imagesValid bool 108 | 109 | // retrieve the downloaded checksums from the bundle 110 | checksums := &types.Artifact{Name: "k3s-sha256sums.txt"} 111 | if err := b.writer.Get(checksums); err != nil { 112 | return err 113 | } 114 | defer checksums.Body.Close() 115 | 116 | // scan the file for the image and binary checksums 117 | scanner := bufio.NewScanner(checksums.Body) 118 | for scanner.Scan() { 119 | 120 | text := scanner.Text() 121 | 122 | // file is structured as " " 123 | spl := strings.Fields(text) 124 | if len(spl) != 2 { 125 | // blank line or a comment 126 | continue 127 | } 128 | shasum, fname := spl[0], spl[1] 129 | 130 | // verify the checksums 131 | switch fname { 132 | case getDownloadAirgapImagesName(opts.Arch): 133 | if opts.ExcludeImages { 134 | imagesValid = true 135 | continue 136 | } 137 | images := &types.Artifact{ 138 | Type: types.ArtifactImages, 139 | Name: "k3s-airgap-images.tar", 140 | } 141 | if err := b.writer.Get(images); err != nil { 142 | return err 143 | } 144 | defer images.Body.Close() 145 | if err := images.Verify(shasum); err != nil { 146 | return err 147 | } 148 | imagesValid = true 149 | case getDownloadK3sBinName(opts.Arch): 150 | k3sbin := &types.Artifact{ 151 | Type: types.ArtifactBin, 152 | Name: "k3s", 153 | } 154 | if err := b.writer.Get(k3sbin); err != nil { 155 | return err 156 | } 157 | defer k3sbin.Body.Close() 158 | if err := k3sbin.Verify(shasum); err != nil { 159 | return err 160 | } 161 | binValid = true 162 | } 163 | } 164 | 165 | if err := scanner.Err(); err != nil && err != io.EOF { 166 | return err 167 | } 168 | 169 | if !binValid || !imagesValid { 170 | return errors.New("A checksum wasn't present for one of the k3s binary or images") 171 | } 172 | 173 | return nil 174 | } 175 | 176 | func getLatestK3sForChannel(channel string) (string, error) { 177 | client := &http.Client{ 178 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 179 | return http.ErrUseLastResponse 180 | }, 181 | } 182 | u := fmt.Sprintf("%s/%s", k3sChannelsRoot, channel) 183 | resp, err := client.Get(u) 184 | if err != nil { 185 | return "", err 186 | } 187 | latestURL := resp.Header.Get("Location") 188 | return path.Base(latestURL), nil 189 | } 190 | 191 | func getDownloadURL(version, component string) string { 192 | return fmt.Sprintf("%s/download/%s/%s", k3sReleasesRootURL, version, component) 193 | } 194 | 195 | func getDownloadChecksumsName(arch string) string { 196 | return fmt.Sprintf("sha256sum-%s.txt", arch) 197 | } 198 | 199 | func getDownloadAirgapImagesName(arch string) string { 200 | return fmt.Sprintf("k3s-airgap-images-%s.tar", arch) 201 | } 202 | 203 | func getDownloadK3sBinName(arch string) string { 204 | var binaryName string 205 | switch arch { 206 | case "amd64": 207 | binaryName = "k3s" 208 | case "arm": 209 | binaryName = "k3s-armhf" 210 | case "arm64": 211 | binaryName = "k3s-arm64" 212 | } 213 | return binaryName 214 | } 215 | -------------------------------------------------------------------------------- /pkg/images/util.go: -------------------------------------------------------------------------------- 1 | package images 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "strings" 9 | "time" 10 | 11 | dockertypes "github.com/docker/docker/api/types" 12 | "github.com/docker/docker/api/types/container" 13 | "github.com/docker/docker/api/types/filters" 14 | "github.com/docker/docker/api/types/strslice" 15 | "github.com/docker/docker/client" 16 | "github.com/docker/go-connections/nat" 17 | 18 | "github.com/tinyzimmer/k3p/pkg/log" 19 | "github.com/tinyzimmer/k3p/pkg/types" 20 | ) 21 | 22 | func getDockerClient() (*client.Client, error) { 23 | return client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) 24 | } 25 | 26 | func filtersForImage(image string) filters.Args { 27 | return filters.NewArgs(filters.Arg("reference", image)) 28 | } 29 | 30 | func sanitizeImageNameSlice(images []string) []string { 31 | out := make([]string, 0) 32 | for _, img := range images { 33 | if sani := sanitizeImageName(img); sani != "" { 34 | out = append(out, sani) 35 | } 36 | } 37 | return out 38 | } 39 | 40 | func sanitizeImageName(image string) string { 41 | imgParts := strings.Split(image, "@") 42 | if len(imgParts) > 1 { 43 | image = imgParts[0] 44 | } 45 | // The leading docker.io messes with image list 46 | if strings.HasPrefix(image, "docker.io/") { 47 | image = strings.TrimPrefix(image, "docker.io/") 48 | } 49 | // Extra check that no empty strings made it in - this check should probably be done somewhere else 50 | return strings.TrimSpace(image) 51 | } 52 | 53 | func ensureImagePulled(cli *client.Client, image, arch string, pullPolicy types.PullPolicy) error { 54 | switch pullPolicy { 55 | case types.PullPolicyNever: 56 | imgs, err := cli.ImageList(context.TODO(), dockertypes.ImageListOptions{ 57 | Filters: filtersForImage(image), 58 | }) 59 | if err != nil { 60 | return err 61 | } 62 | if len(imgs) == 0 { 63 | return fmt.Errorf("Image %s is not present on the machine", image) 64 | } 65 | case types.PullPolicyIfNotPresent: 66 | log.Debug("Checking local docker images for", image) 67 | imgs, err := cli.ImageList(context.TODO(), dockertypes.ImageListOptions{ 68 | Filters: filtersForImage(image), 69 | }) 70 | if err != nil { 71 | log.Debugf("Error trying to list images for %s: %s\n", image, err.Error()) 72 | } 73 | if imgs == nil || len(imgs) != 1 { 74 | return pullImage(cli, image, arch) 75 | } 76 | log.Infof("Image %s already present on the machine\n", image) 77 | case types.PullPolicyAlways: 78 | return pullImage(cli, image, arch) 79 | } 80 | return nil 81 | } 82 | 83 | func pullImage(cli *client.Client, image, arch string) error { 84 | log.Infof("Pulling image for %s\n", image) 85 | rdr, err := cli.ImagePull(context.TODO(), image, dockertypes.ImagePullOptions{Platform: arch}) 86 | if err != nil { 87 | return err 88 | } 89 | log.LevelReader(log.LevelDebug, rdr) 90 | return nil 91 | } 92 | 93 | func registryContainerConfigs() (*container.Config, *container.HostConfig) { 94 | // Expose a random local port to the registry 95 | exposedPorts, portBindings, err := nat.ParsePortSpecs([]string{"0:5000"}) 96 | if err != nil { 97 | log.Fatal(err) 98 | } 99 | containerConig := &container.Config{ 100 | Image: "registry:2", 101 | ExposedPorts: exposedPorts, 102 | Volumes: map[string]struct{}{ 103 | "/var/lib/registry": struct{}{}, 104 | }, 105 | } 106 | hostConfig := &container.HostConfig{ 107 | PortBindings: portBindings, 108 | } 109 | return containerConig, hostConfig 110 | } 111 | 112 | func registryVolumeContainerConfigs(regsitryContainerID string) (*container.Config, *container.HostConfig) { 113 | containerConfig := &container.Config{ 114 | Image: "busybox", 115 | Volumes: map[string]struct{}{ 116 | "/var/lib/registry": struct{}{}, 117 | }, 118 | Cmd: strslice.StrSlice([]string{ 119 | "tar", "-cvz", "--file=/var/registry-data.tgz", "--directory=/var/lib/registry", ".", 120 | }), // |- should be constant -| 121 | } 122 | hostConfig := &container.HostConfig{ 123 | VolumesFrom: []string{regsitryContainerID}, 124 | } 125 | return containerConfig, hostConfig 126 | } 127 | 128 | func createAndStartContainer(cli *client.Client, containerConfig *container.Config, hostConfig *container.HostConfig) (id string, err error) { 129 | cont, err := cli.ContainerCreate(context.TODO(), containerConfig, hostConfig, nil, "") 130 | if err != nil { 131 | return "", err 132 | } 133 | if err := cli.ContainerStart(context.TODO(), cont.ID, dockertypes.ContainerStartOptions{}); err != nil { 134 | defer func() { 135 | if cerr := cli.ContainerRemove(context.TODO(), cont.ID, dockertypes.ContainerRemoveOptions{ 136 | Force: true, 137 | RemoveVolumes: true, 138 | }); cerr != nil { 139 | log.Warning("Error removing failed container:", cerr) 140 | } 141 | }() 142 | return "", err 143 | } 144 | return cont.ID, nil 145 | } 146 | 147 | func getHostPortForContainer(cli *client.Client, containerID string, portProto string) (string, error) { 148 | deets, err := cli.ContainerInspect(context.TODO(), containerID) 149 | if err != nil { 150 | return "", err 151 | } 152 | localPortMap, ok := deets.NetworkSettings.Ports["5000/tcp"] 153 | if !ok { 154 | return "", fmt.Errorf("Could not determine host port for %s on %s from %+v", portProto, containerID, deets.HostConfig.PortBindings) 155 | } 156 | localPort := localPortMap[0].HostPort 157 | return localPort, nil 158 | } 159 | 160 | func waitForLocalRegistry(port string, timeout time.Duration) error { 161 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 162 | defer cancel() 163 | client := &http.Client{Timeout: time.Second * 2} 164 | for { 165 | select { 166 | case <-ctx.Done(): 167 | return errors.New("Time out reached waiting for registry to be ready") 168 | default: 169 | res, err := client.Get(fmt.Sprintf("http://localhost:%s/v2/_catalog", port)) 170 | if err != nil { 171 | log.Debug("Error waiting for registry to be ready, will retry:", err) 172 | continue 173 | } 174 | if res.StatusCode != http.StatusOK { 175 | log.Debug("Non-200 status code from registry catalog, will retry:", res.StatusCode) 176 | continue 177 | } 178 | log.Debug("Local registry is ready") 179 | return nil 180 | } 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /pkg/install/installer.go: -------------------------------------------------------------------------------- 1 | package install 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | "os/exec" 12 | "path" 13 | "strings" 14 | "time" 15 | 16 | "github.com/tinyzimmer/k3p/pkg/images/registry" 17 | "github.com/tinyzimmer/k3p/pkg/log" 18 | "github.com/tinyzimmer/k3p/pkg/types" 19 | "github.com/tinyzimmer/k3p/pkg/util" 20 | ) 21 | 22 | // New returns a new package installer. 23 | func New() types.Installer { return &installer{} } 24 | 25 | type installer struct{} 26 | 27 | func (i *installer) Install(target types.Node, pkg types.Package, opts *types.InstallOptions) error { 28 | defer pkg.Close() 29 | 30 | log.Info("Copying the archive to the rancher installation directory") 31 | 32 | archive, err := pkg.Archive() 33 | if err != nil { 34 | return err 35 | } 36 | if err := target.WriteFile(archive.Reader(), types.InstalledPackageFile, "0644", archive.Size()); err != nil { 37 | return err 38 | } 39 | 40 | // retrieve the meta to see if there is a EULA 41 | meta := pkg.GetMeta() 42 | 43 | if meta.Manifest.HasEULA() { 44 | // check package for a EULA 45 | eula := &types.Artifact{Name: types.ManifestEULAFile} 46 | if err := pkg.Get(eula); err == nil { 47 | // EULA found 48 | if err := promptEULA(eula, opts.AcceptEULA); err != nil { 49 | return err 50 | } 51 | } else if !os.IsNotExist(err) { 52 | // Error other than file not found 53 | return err 54 | } 55 | } 56 | 57 | cfg := pkg.GetMeta().DeepCopy().GetPackageConfig() 58 | log.Debugf("Package configuration: %+v\n", cfg) 59 | if cfg != nil { 60 | if err := cfg.ApplyVariables(opts.Variables); err != nil { 61 | return err 62 | } 63 | } 64 | execOpts := opts.ToExecOpts(cfg) 65 | 66 | if opts.InitHA { 67 | // append --cluster-init 68 | execOpts.Env["INSTALL_K3S_EXEC"] = execOpts.Env["INSTALL_K3S_EXEC"] + " --cluster-init" 69 | // Check if we need to generate an HA token 70 | if opts.NodeToken == "" { 71 | log.Info("Generating a node token for additional control-plane instances") 72 | token := util.GenerateToken(128) 73 | log.Debugf("Writing the contents of the server token to %s\n", types.ServerTokenFile) 74 | if err := target.WriteFile(ioutil.NopCloser(strings.NewReader(token)), types.ServerTokenFile, "0600", 128); err != nil { 75 | return err 76 | } 77 | execOpts.Env["K3S_TOKEN"] = token 78 | execOpts.Secrets = append(execOpts.Secrets, token) 79 | } 80 | } 81 | 82 | if meta.ImageBundleFormat == types.ImageBundleRegistry { 83 | log.Info("Package was generated with private registry") 84 | if err := setupPrivateRegistry(target, meta, opts); err != nil { 85 | return err 86 | } 87 | } 88 | 89 | installedConfig := &types.InstallConfig{InstallOptions: opts} 90 | log.Debugf("Built installation config %+v\n", installedConfig) 91 | 92 | // unpack the manifest onto the node 93 | if err := util.SyncPackageToNode(target, pkg, installedConfig); err != nil { 94 | return err 95 | } 96 | 97 | // Install K3s 98 | if target.GetType() != types.NodeDocker { 99 | // let's not lie to the user when we are doing docker installs 100 | log.Info("Running k3s installation script") 101 | } 102 | return target.Execute(execOpts) 103 | } 104 | 105 | func promptEULA(eula *types.Artifact, autoAccept bool) error { 106 | if autoAccept { 107 | return nil 108 | } 109 | pager := os.Getenv("PAGER") 110 | if pager == "" { 111 | pager = "less" 112 | } 113 | cmd := exec.Command(pager) 114 | cmd.Stdin = eula.Body 115 | cmd.Stdout = os.Stdout 116 | if err := cmd.Run(); err != nil { 117 | return err 118 | } 119 | scanner := bufio.NewScanner(os.Stdin) 120 | for { 121 | fmt.Print("Do you accept the terms of the EULA? [y/N] ") 122 | scanner.Scan() 123 | text := scanner.Text() 124 | switch strings.ToLower(text) { 125 | case "y": 126 | time.Sleep(time.Second) 127 | return nil 128 | case "n": 129 | return errors.New("EULA was declined") 130 | default: 131 | fmt.Printf("%q is not a valid response, choose 'y' or 'n' \n", text) 132 | } 133 | } 134 | } 135 | 136 | func setupPrivateRegistry(target types.Node, meta *types.PackageMeta, opts *types.InstallOptions) error { 137 | registryManifestPath := path.Join(types.K3sManifestsDir, "private-registry") 138 | 139 | log.Info("Setting up registry TLS") 140 | caCert, secrets, err := registry.GenerateRegistryTLSSecrets(&types.RegistryTLSOptions{ 141 | Name: meta.GetName(), 142 | RegistryTLSCertFile: opts.RegistryTLSCertFile, 143 | RegistryTLSKeyFile: opts.RegistryTLSKeyFile, 144 | RegistryTLSCAFile: opts.RegistryTLSCAFile, 145 | }) 146 | if err != nil { 147 | return err 148 | } 149 | if err := target.WriteFile(nopCloser(caCert), registry.RegistryCAPath, "0644", size(caCert)); err != nil { 150 | return err 151 | } 152 | if err := target.WriteFile(nopCloser(secrets), path.Join(registryManifestPath, "registry-tls-secrets.yaml"), "0644", size(secrets)); err != nil { 153 | return err 154 | } 155 | 156 | log.Info("Writing secrets for registry authentication") 157 | if opts.RegistrySecret == "" { 158 | log.Info("Generating password for registry authentication") 159 | opts.RegistrySecret = util.GenerateToken(16) 160 | } 161 | authSecret, err := registry.GenerateRegistryAuthSecret(opts.RegistrySecret) 162 | if err != nil { 163 | return err 164 | } 165 | if err := target.WriteFile(nopCloser(authSecret), path.Join(registryManifestPath, "registry-auth-secret.yaml"), "0644", size(authSecret)); err != nil { 166 | return err 167 | } 168 | 169 | log.Info("Writing deployments and services for the private registry") 170 | svcs, err := registry.GenerateRegistryServices(opts.GetRegistryNodePort()) 171 | if err != nil { 172 | return err 173 | } 174 | deployments, err := registry.GenerateRegistryDeployments(meta.GetRegistryImageName()) 175 | if err != nil { 176 | return err 177 | } 178 | if err := target.WriteFile(nopCloser(svcs), path.Join(registryManifestPath, "registry-services.yaml"), "0644", size(svcs)); err != nil { 179 | return err 180 | } 181 | if err := target.WriteFile(nopCloser(deployments), path.Join(registryManifestPath, "registry-deployments.yaml"), "0644", size(deployments)); err != nil { 182 | return err 183 | } 184 | 185 | log.Info("Writing containerd configuration for the private registry") 186 | registryConf, err := registry.GenerateRegistriesYaml(opts.RegistrySecret, opts.GetRegistryNodePort()) 187 | if err != nil { 188 | return err 189 | } 190 | if err := target.WriteFile(nopCloser(registryConf), types.K3sRegistriesYamlPath, "0644", size(registryConf)); err != nil { 191 | return err 192 | } 193 | 194 | return nil 195 | } 196 | 197 | func size(b []byte) int64 { return int64(len(b)) } 198 | 199 | func nopCloser(b []byte) io.ReadCloser { return ioutil.NopCloser(bytes.NewReader(b)) } 200 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # k3p 2 | 3 | A `k3s` packager and installer, originally and primarily intended for air-gapped or restricted-access deployments, but could satisfy other use cases as well. 4 | 5 | For documentation on `k3p` usage, see the [command docs here](doc/k3p.md). 6 | 7 | ## TODO: 8 | 9 | - Per node configs (can't decide if worth it) 10 | - Conditional helm charts? 11 | - Docs 12 | 13 | ## Quickstart 14 | 15 | First download a binary for your system from the [releases](https://github.com/tinyzimmer/k3p/releases) page. 16 | 17 | Or, to build from source, on a system with, `make`, `git`, and `go` installed. 18 | 19 | ```bash 20 | git clone https://github.com/tinyzimmer/k3p 21 | cd k3p 22 | make install 23 | 24 | # If you do not have make installed, you can build and install manually with: 25 | cd cmd/k3p 26 | go build -o $(go env GOPATH)/bin/k3p . 27 | ``` 28 | 29 | If you'd like to build a self-installing package currently you need a linux machine. This is until I can publish public releases. 30 | To get around this on other platforms for now you can build the docker image and use it like this: 31 | 32 | ```bash 33 | $ make docker IMG=k3p 34 | # ... 35 | $ docker run --rm \ 36 | -v /var/run/docker.sock:/var/run/docker.sock \ 37 | -v ~/.k3p:/root/.k3p \ 38 | -v /path/to/manifests:/manifests \ 39 | k3p build --run-file --compress 40 | ``` 41 | 42 | You can build a package with the `build` command. By default it will scan your current directory, and 43 | detect objects to be included in the archive. See the usage documentation for other configuration options. 44 | 45 | ```bash 46 | $ k3p build 47 | 2020/12/10 10:49:37 [INFO] Generated name for package "intelligent_wu" 48 | 2020/12/10 10:49:37 [INFO] Detecting latest k3s version for channel stable 49 | 2020/12/10 10:49:38 [INFO] Latest k3s version is v1.19.4+k3s1 50 | 2020/12/10 10:49:38 [INFO] Packaging distribution for version "v1.19.4+k3s1" using "amd64" architecture 51 | 2020/12/10 10:49:38 [INFO] Downloading core k3s components 52 | 2020/12/10 10:49:38 [INFO] Fetching checksums... 53 | 2020/12/10 10:49:38 [INFO] Fetching k3s install script... 54 | 2020/12/10 10:49:38 [INFO] Fetching k3s binary... 55 | 2020/12/10 10:49:38 [INFO] Fetching k3s airgap images... 56 | 2020/12/10 10:49:38 [INFO] Validating checksums... 57 | 2020/12/10 10:49:40 [INFO] Searching for kubernetes manifests to include in the archive 58 | 2020/12/10 10:49:40 [INFO] Detected kubernetes manifest: "/home/tinyzimmer/devel/k3p/example-manifests/whoami.yaml" 59 | 2020/12/10 10:49:40 [INFO] Parsing kubernetes manifests for container images to download 60 | 2020/12/10 10:49:40 [INFO] Found appsv1 Deployment: whoami 61 | 2020/12/10 10:49:40 [INFO] Detected the following images to bundle with the package: [traefik/whoami:latest] 62 | 2020/12/10 10:49:40 [INFO] Pulling image for traefik/whoami:latest 63 | 2020/12/10 10:49:42 [INFO] Adding container images to package 64 | 2020/12/10 10:49:42 [INFO] Writing package metadata 65 | 2020/12/10 10:49:42 [INFO] Archiving version "latest" of "intelligent_wu" to "/home/tinyzimmer/devel/k3p/package.tar" 66 | ``` 67 | 68 | You can optionally exclude images for the archive, however this is not the default behavior as this project was originally intended 69 | for fully airgapped deployments. Any raw kubernetes `yaml` or `helm` charts found (that are not excluded) will be included and applied 70 | automatically upon installation. 71 | 72 | You can then install the package to a system using the `install` command. Installations can be performed either on the local system (requires root), 73 | over a remote SSH connection (requires SSH user have passwordless `sudo`), or to docker containers on the local system similar to [`k3d`](https://github.com/rancher/k3d). 74 | 75 | Again, see the usage documentation for more configuration options, and how to use the various installation modes, but to just install to the local system: 76 | 77 | ```bash 78 | # Will work on any linux system (and WSL2 using genie) 79 | $ sudo k3p install package.tar 80 | 2020/12/10 10:57:56 [INFO] Loading the archive 81 | 2020/12/10 10:57:57 [INFO] Copying the archive to the rancher installation directory 82 | 2020/12/10 10:57:58 [INFO] Installing binaries to /usr/local/bin 83 | 2020/12/10 10:57:58 [INFO] Installing scripts to /usr/local/bin/k3p-scripts 84 | 2020/12/10 10:57:58 [INFO] Installing images to /var/lib/rancher/k3s/agent/images 85 | 2020/12/10 10:57:59 [INFO] Installing manifests to /var/lib/rancher/k3s/server/manifests 86 | 2020/12/10 10:57:59 [INFO] Running k3s installation script 87 | 2020/12/10 10:57:59 [K3S] [INFO] Skipping k3s download and verify 88 | 2020/12/10 10:57:59 [K3S] [INFO] Skipping installation of SELinux RPM 89 | 2020/12/10 10:57:59 [K3S] [INFO] Skipping /usr/local/bin/kubectl symlink to k3s, command exists in PATH at /usr/bin/kubectl 90 | 2020/12/10 10:57:59 [K3S] [INFO] Creating /usr/local/bin/crictl symlink to k3s 91 | 2020/12/10 10:57:59 [K3S] [INFO] Creating /usr/local/bin/ctr symlink to k3s 92 | 2020/12/10 10:57:59 [K3S] [INFO] Creating killall script /usr/local/bin/k3s-killall.sh 93 | 2020/12/10 10:57:59 [K3S] [INFO] Creating uninstall script /usr/local/bin/k3s-uninstall.sh 94 | 2020/12/10 10:57:59 [K3S] [INFO] env: Creating environment file /etc/systemd/system/k3s.service.env 95 | 2020/12/10 10:57:59 [K3S] [INFO] systemd: Creating service file /etc/systemd/system/k3s.service 96 | 2020/12/10 10:57:59 [K3S] [INFO] systemd: Enabling k3s unit 97 | 2020/12/10 10:57:59 [K3S] Created symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service. 98 | 2020/12/10 10:57:59 [K3S] [INFO] systemd: Starting k3s 99 | 2020/12/10 10:58:06 [INFO] The cluster has been installed 100 | 2020/12/10 10:58:06 [INFO] You can view the cluster by running `k3s kubectl cluster-info` 101 | 102 | $ sudo k3s kubectl cluster-info 103 | Kubernetes master is running at https://127.0.0.1:6443 104 | CoreDNS is running at https://127.0.0.1:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy 105 | Metrics-server is running at https://127.0.0.1:6443/api/v1/namespaces/kube-system/services/https:metrics-server:/proxy 106 | 107 | To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'. 108 | 109 | $ sudo k3s kubectl get pod 110 | NAME READY STATUS RESTARTS AGE 111 | whoami-5dc4dd9cdf-qvvnz 1/1 Running 0 32s 112 | ``` 113 | 114 | For further information on adding worker nodes and/or setting up HA, you can view the command documentation, 115 | however more complete documentation will come in the future in the form of [examples](examples/) and other docs. 116 | There are already a few simple examples that you can use to get a general understanding of the workflow. -------------------------------------------------------------------------------- /pkg/images/registry/templates.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "bytes" 5 | "text/template" 6 | 7 | "github.com/Masterminds/sprig" 8 | ) 9 | 10 | func executeTemplate(tmpl *template.Template, vars map[string]interface{}) ([]byte, error) { 11 | var buf bytes.Buffer 12 | if err := tmpl.Execute(&buf, vars); err != nil { 13 | return nil, err 14 | } 15 | return buf.Bytes(), nil 16 | } 17 | 18 | var registryServicesTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(`--- 19 | apiVersion: v1 20 | kind: Service 21 | metadata: 22 | name: {{ .RegistryK8sAppName }} 23 | namespace: {{ .RegistryNamespace }} 24 | labels: 25 | k8s-app: {{ .RegistryK8sAppName }} 26 | spec: 27 | type: NodePort 28 | selector: 29 | k8s-app: {{ .RegistryK8sAppName }} 30 | ports: 31 | - port: 5000 32 | protocol: TCP 33 | targetPort: 5000 34 | nodePort: {{ .RegistryNodePort }} 35 | --- 36 | apiVersion: v1 37 | kind: Service 38 | metadata: 39 | name: {{ .KubenabK8sAppName }} 40 | namespace: {{ .RegistryNamespace }} 41 | labels: 42 | k8s-app: {{ .KubenabK8sAppName }} 43 | spec: 44 | selector: 45 | k8s-app: {{ .KubenabK8sAppName }} 46 | type: ClusterIP 47 | ports: 48 | - port: 443 49 | protocol: "TCP" 50 | name: https 51 | `)) 52 | 53 | var registriesYamlTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(` 54 | mirrors: 55 | registry.private: 56 | endpoint: 57 | - https://localhost:{{ .RegistryNodePort }} 58 | 59 | configs: 60 | "localhost:{{ .RegistryNodePort }}": 61 | auth: 62 | username: {{ .Username }} 63 | password: {{ .Password }} 64 | tls: 65 | ca_file: {{ .RegistryCAPath }} 66 | `)) 67 | 68 | var registryAuthSecretTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(` 69 | apiVersion: v1 70 | kind: Secret 71 | metadata: 72 | name: {{ .RegistryAuthSecret }} 73 | namespace: {{ .RegistryNamespace }} 74 | labels: 75 | k8s-app: {{ .RegistryK8sAppName }} 76 | type: Opaque 77 | data: 78 | htpasswd: {{ .RegistryAuthHtpasswd | b64enc }} 79 | `)) 80 | 81 | var registryTLSTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(`--- 82 | apiVersion: v1 83 | kind: Secret 84 | metadata: 85 | name: {{ .RegistryTLSSecret }} 86 | namespace: {{ .RegistryNamespace }} 87 | labels: 88 | k8s-app: {{ .RegistryK8sAppName }} 89 | type: kubernetes.io/tls 90 | data: 91 | tls.crt: {{ .TLSCertificate | b64enc }} 92 | tls.key: {{ .TLSPrivateKey | b64enc }} 93 | ca.crt: {{ .TLSCACertificate | b64enc }} 94 | --- 95 | apiVersion: v1 96 | kind: Secret 97 | metadata: 98 | name: {{ .KubenabTLSSecret }} 99 | namespace: {{ .RegistryNamespace }} 100 | labels: 101 | k8s-app: {{ .KubenabK8sAppName }} 102 | type: kubernetes.io/tls 103 | data: 104 | tls.crt: {{ .TLSCertificate | b64enc }} 105 | tls.key: {{ .TLSPrivateKey | b64enc }} 106 | ca.crt: {{ .TLSCACertificate | b64enc }} 107 | --- 108 | apiVersion: admissionregistration.k8s.io/v1beta1 109 | kind: MutatingWebhookConfiguration 110 | metadata: 111 | name: kubenab-mutate 112 | webhooks: 113 | - name: kubenab-mutate.kubenab.com 114 | objectSelector: 115 | matchExpressions: 116 | - key: k8s-app 117 | operator: NotIn 118 | values: ["kube-dns", "kubenab", "private-registry", "metrics-server"] 119 | rules: 120 | - operations: [ "CREATE", "UPDATE" ] 121 | apiGroups: [""] 122 | apiVersions: ["v1"] 123 | resources: ["pods"] 124 | failurePolicy: Fail 125 | clientConfig: 126 | service: 127 | name: kubenab 128 | namespace: kube-system 129 | path: "/mutate" 130 | caBundle: {{ .TLSCACertificate | b64enc }} 131 | `)) 132 | 133 | var registryDeploymentsTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(`--- 134 | apiVersion: apps/v1 135 | kind: Deployment 136 | metadata: 137 | name: {{ .RegistryK8sAppName }} 138 | namespace: {{ .RegistryNamespace }} 139 | labels: 140 | k8s-app: {{ .RegistryK8sAppName }} 141 | spec: 142 | replicas: 1 143 | selector: 144 | matchLabels: 145 | k8s-app: {{ .RegistryK8sAppName }} 146 | template: 147 | metadata: 148 | labels: 149 | k8s-app: {{ .RegistryK8sAppName }} 150 | spec: 151 | priorityClassName: system-cluster-critical 152 | volumes: 153 | - name: registry-data 154 | emptyDir: {} 155 | - name: {{ .RegistryTLSSecret }} 156 | secret: 157 | secretName: {{ .RegistryTLSSecret }} 158 | - name: {{ .RegistryAuthSecret }} 159 | secret: 160 | secretName: {{ .RegistryAuthSecret }} 161 | initContainers: 162 | - name: data-extractor 163 | image: {{ .RegistryDataImage }} 164 | imagePullPolicy: Never 165 | command: ['tar', '-xvz', '--file=/var/registry-data.tgz', '--directory=/var/lib/registry'] 166 | volumeMounts: 167 | - name: registry-data 168 | mountPath: /var/lib/registry 169 | containers: 170 | - name: {{ .RegistryK8sAppName }} 171 | image: registry:2 172 | imagePullPolicy: Never 173 | env: 174 | - name: REGISTRY_HTTP_TLS_CERTIFICATE 175 | value: /etc/tls/certs/tls.crt 176 | - name: REGISTRY_HTTP_TLS_KEY 177 | value: /etc/tls/certs/tls.key 178 | - name: REGISTRY_AUTH 179 | value: htpasswd 180 | - name: REGISTRY_AUTH_HTPASSWD_REALM 181 | value: "Private Registry Realm" 182 | - name: REGISTRY_AUTH_HTPASSWD_PATH 183 | value: /etc/auth/htpasswd 184 | ports: 185 | - containerPort: 5000 186 | volumeMounts: 187 | - name: registry-data 188 | mountPath: /var/lib/registry 189 | - name: {{ .RegistryTLSSecret }} 190 | mountPath: /etc/tls/certs 191 | readOnly: true 192 | - name: {{ .RegistryAuthSecret }} 193 | mountPath: /etc/auth 194 | readOnly: true 195 | --- 196 | apiVersion: apps/v1 197 | kind: Deployment 198 | metadata: 199 | name: {{ .KubenabK8sAppName }} 200 | namespace: {{ .RegistryNamespace }} 201 | labels: 202 | k8s-app: {{ .KubenabK8sAppName }} 203 | spec: 204 | selector: 205 | matchLabels: 206 | k8s-app: {{ .KubenabK8sAppName }} 207 | replicas: 1 208 | template: 209 | metadata: 210 | labels: 211 | k8s-app: {{ .KubenabK8sAppName }} 212 | spec: 213 | priorityClassName: system-cluster-critical 214 | containers: 215 | - name: {{ .KubenabK8sAppName }} 216 | image: {{ .KubenabImage }} 217 | imagePullPolicy: Never 218 | env: 219 | - name: DOCKER_REGISTRY_URL 220 | value: "registry.private" 221 | - name: WHITELIST_NAMESPACES 222 | value: "kube-system" 223 | - name: WHITELIST_REGISTRIES 224 | value: "registry.private,rancher" 225 | - name: REPLACE_REGISTRY_URL 226 | value: "true" 227 | ports: 228 | - containerPort: 443 229 | name: https 230 | volumeMounts: 231 | - name: {{ .KubenabTLSSecret }} 232 | mountPath: /etc/admission-controller/tls 233 | volumes: 234 | - name: {{ .KubenabTLSSecret }} 235 | secret: 236 | secretName: {{ .KubenabTLSSecret }} 237 | `)) 238 | -------------------------------------------------------------------------------- /examples/docker/README.md: -------------------------------------------------------------------------------- 1 | # Playing with Docker 2 | 3 | This directory goes into more detail on smoke testing packages with docker. 4 | If you have used [`k3d`](https://github.com/rancher/k3d) in the past most of this will be familiar to you. 5 | 6 | To start off, build the package in this directory (for the purpose of these examples we'll exclude images from the archive): 7 | 8 | ```bash 9 | # Build the package and give it a unique name 10 | $ k3p build --exclude-images --name=k3p-docker 11 | 12 | 2020/12/14 10:09:59 [INFO] Building package "k3p-docker" 13 | 2020/12/14 10:09:59 [INFO] Detecting latest k3s version for channel stable 14 | 2020/12/14 10:10:00 [INFO] Latest k3s version is v1.19.4+k3s1 15 | 2020/12/14 10:10:00 [INFO] Packaging distribution for version "v1.19.4+k3s1" using "amd64" architecture 16 | 2020/12/14 10:10:00 [INFO] Downloading core k3s components 17 | 2020/12/14 10:10:00 [INFO] Fetching checksums... 18 | 2020/12/14 10:10:00 [INFO] Fetching k3s install script... 19 | 2020/12/14 10:10:00 [INFO] Fetching k3s binary... 20 | 2020/12/14 10:10:00 [INFO] Skipping bundling k3s airgap images with the package 21 | 2020/12/14 10:10:00 [INFO] Validating checksums... 22 | 2020/12/14 10:10:00 [INFO] Searching "/home/tinyzimmer/devel/k3p/examples/docker" for kubernetes manifests to include in the archive 23 | 2020/12/14 10:10:00 [INFO] Detected kubernetes manifest: "/home/tinyzimmer/devel/k3p/examples/docker/whoami.yaml" 24 | 2020/12/14 10:10:00 [INFO] Skipping bundling container images with the package 25 | 2020/12/14 10:10:00 [INFO] Writing package metadata 26 | 2020/12/14 10:10:00 [INFO] Archiving version "latest" of "k3p-docker" to "/home/tinyzimmer/devel/k3p/examples/docker/package.tar" 27 | ``` 28 | 29 | To install this package to a simple single node cluster running in docker you can do the following: 30 | 31 | ```bash 32 | # --write-kubeconfig is optional and will extract the kubeconfig once the server is up 33 | # otherwise instructions are printed for fetching it directly from the container 34 | $ k3p install package.tar --docker --write-kubeconfig kubeconfig.yaml 35 | 2020/12/14 10:11:13 [INFO] Loading the archive 36 | 2020/12/14 10:11:13 [INFO] Creating docker network k3p-docker 37 | 2020/12/14 10:11:13 [INFO] Creating docker volume k3p-docker-server-0 38 | 2020/12/14 10:11:14 [INFO] Copying the archive to the rancher installation directory 39 | 2020/12/14 10:11:14 [INFO] Installing binaries to /usr/local/bin 40 | 2020/12/14 10:11:14 [INFO] Installing scripts to /usr/local/bin/k3p-scripts 41 | 2020/12/14 10:11:14 [INFO] Installing manifests to /var/lib/rancher/k3s/server/manifests 42 | 2020/12/14 10:11:14 [INFO] Running k3s installation script 43 | 2020/12/14 10:11:14 [INFO] Starting k3s docker node k3p-docker-server-0 44 | 2020/12/14 10:11:15 [INFO] Starting k3s docker node k3p-docker-serverlb 45 | 2020/12/14 10:11:15 [INFO] Waiting for server to write the admin kubeconfig 46 | 2020/12/14 10:11:17 [INFO] Writing the kubeconfig to "kubeconfig.yaml" 47 | 2020/12/14 10:11:17 [INFO] The cluster has been installed 48 | 2020/12/14 10:11:17 [INFO] You can view the cluster by running `kubectl --kubeconfig kubeconfig.yaml cluster-info` 49 | 50 | $ kubectl --kubeconfig kubeconfig.yaml cluster-info 51 | Kubernetes master is running at https://127.0.0.1:6443 52 | CoreDNS is running at https://127.0.0.1:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy 53 | Metrics-server is running at https://127.0.0.1:6443/api/v1/namespaces/kube-system/services/https:metrics-server:/proxy 54 | 55 | To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'. 56 | 57 | $ kubectl --kubeconfig kubeconfig.yaml get pod 58 | NAME READY STATUS RESTARTS AGE 59 | whoami-5db874f58d-dcx48 1/1 Running 0 54s 60 | 61 | # To remove the cluster when you are done 62 | $ k3p uninstall --name=k3p-docker # The --name flag supports tab completion 63 | 2020/12/14 10:13:53 [INFO] Removing docker cluster k3p-docker 64 | 2020/12/14 10:13:53 [INFO] Removing docker container and volumes for k3p-docker-serverlb 65 | 2020/12/14 10:13:53 [INFO] Removing docker container and volumes for k3p-docker-server-0 66 | 2020/12/14 10:13:54 [INFO] Removing docker network k3p-docker 67 | ``` 68 | 69 | You can specify server/agent count and configurations also (this is the same as for a regular install) 70 | 71 | ```bash 72 | $ k3p install package.tar --docker --write-kubeconfig kubeconfig.yaml \ 73 | --servers 3 --agents 3 \ # Specify number of server and agent nodes 74 | --k3s-server-arg="--disable=traefik" # can be specified multiple times, there is also an agent equivalent 75 | 76 | # ... 77 | # ... 78 | 79 | $ kubectl --kubeconfig kubeconfig.yaml get node 80 | NAME STATUS ROLES AGE VERSION 81 | k3p-docker-agent-0 Ready worker 47s v1.19.4+k3s1 82 | k3p-docker-agent-1 Ready worker 50s v1.19.4+k3s1 83 | k3p-docker-agent-2 Ready worker 49s v1.19.4+k3s1 84 | k3p-docker-server-0 Ready etcd,master 55s v1.19.4+k3s1 85 | k3p-docker-server-1 Ready etcd,master 24s v1.19.4+k3s1 86 | k3p-docker-server-2 Ready etcd,master 44s v1.19.4+k3s1 87 | 88 | $ kubectl --kubeconfig kubeconfig.yaml get pod -A 89 | NAMESPACE NAME READY STATUS RESTARTS AGE 90 | default whoami-5db874f58d-xtgrl 1/1 Running 0 2m42s 91 | kube-system coredns-66c464876b-sr4fw 1/1 Running 0 2m42s 92 | kube-system local-path-provisioner-7ff9579c6-5wpnq 1/1 Running 0 2m42s 93 | kube-system metrics-server-7b4f8b595-rmwl5 1/1 Running 0 2m42s 94 | ``` 95 | 96 | Forwarding ports to specific nodes in the cluster works the same as `k3d` 97 | 98 | ```bash 99 | $ k3p install package.tar --docker \ 100 | --publish 8080:80@loadbalancer \ # Forward 8080 on the local machine to 80 on the LoadBalancer 101 | --publish 8081:80@server[0] # Forward 8081 on the local machine to 80 on the first server instance 102 | 103 | 2020/12/14 10:29:10 [INFO] Loading the archive 104 | 2020/12/14 10:29:10 [INFO] Creating docker network k3p-docker 105 | 2020/12/14 10:29:10 [INFO] Creating docker volume k3p-docker-server-0 106 | 2020/12/14 10:29:10 [INFO] Copying the archive to the rancher installation directory 107 | 2020/12/14 10:29:10 [INFO] Installing binaries to /usr/local/bin 108 | 2020/12/14 10:29:10 [INFO] Installing scripts to /usr/local/bin/k3p-scripts 109 | 2020/12/14 10:29:10 [INFO] Installing manifests to /var/lib/rancher/k3s/server/manifests 110 | 2020/12/14 10:29:11 [INFO] Running k3s installation script 111 | 2020/12/14 10:29:11 [INFO] Starting k3s docker node k3p-docker-server-0 112 | 2020/12/14 10:29:11 [INFO] Starting k3s docker node k3p-docker-serverlb 113 | 2020/12/14 10:29:12 [INFO] The cluster has been installed 114 | 2020/12/14 10:29:12 [INFO] You can retrieve the kubeconfig by running `docker cp k3p-docker-server-0:/etc/rancher/k3s/k3s.yaml ./kubeconfig.yaml` 115 | 116 | $ docker ps 117 | CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 118 | b90d5ac9107a rancher/k3d-proxy:latest "/bin/sh -c nginx-pr…" 7 seconds ago Up 5 seconds 0.0.0.0:6443->6443/tcp, 0.0.0.0:8080->80/tcp k3p-docker-serverlb 119 | f9d226f0a17b rancher/k3s:v1.19.4-k3s1 "/bin/k3s server --t…" 7 seconds ago Up 6 seconds 0.0.0.0:8081->80/tcp k3p-docker-server-0 120 | ``` --------------------------------------------------------------------------------