├── .semgrepignore ├── local-testing ├── templates ├── main_test.go └── .gitignore ├── .dockerignore ├── e2e ├── templates │ ├── kube-proxy-values.yaml.tmpl │ ├── kubeconfig.tmpl │ ├── kubernetes-values.yaml.tmpl │ └── kube-apiserver-values.yaml.tmpl ├── generate-secrets.sh ├── 98-ssh-as-root-with-password ├── Dockerfile.openssh-server ├── docker-compose.yml └── Dockerfile ├── pkg ├── kubernetes │ └── client │ │ ├── patches.go │ │ ├── clientset_test.go │ │ ├── clientset.go │ │ ├── getter_test.go │ │ ├── getter.go │ │ ├── client_test.go │ │ ├── kubeconfig.go │ │ └── client.go ├── helm │ └── release │ │ ├── client.go │ │ ├── release_examples_test.go │ │ ├── release_internal_test.go │ │ └── release_test.go ├── etcd │ ├── fake.go │ └── member_test.go ├── pki │ ├── privatekey.go │ ├── privatekey_test.go │ ├── etcd_test.go │ └── etcd.go ├── host │ ├── transport │ │ ├── transport.go │ │ ├── direct │ │ │ ├── direct.go │ │ │ └── direct_test.go │ │ └── ssh │ │ │ └── defaults.go │ ├── host.go │ └── host_test.go ├── defaults │ └── defaults.go ├── controlplane │ ├── validate_test.go │ ├── validate.go │ ├── kube-scheduler_test.go │ ├── kube-scheduler.go │ ├── controlplane_test.go │ ├── kube-controller-manager_test.go │ └── kube-apiserver_test.go ├── types │ ├── certificate.go │ ├── certificate_test.go │ ├── privatekey.go │ ├── resource.go │ └── privatekey_test.go ├── apiloadbalancer │ ├── api-loadbalancer_test.go │ ├── api-loadbalancers_integration_test.go │ └── api-loadbalancers_test.go ├── container │ ├── runtime │ │ ├── runtime.go │ │ ├── fake.go │ │ └── docker │ │ │ └── fake.go │ ├── resource │ │ └── resource.go │ ├── hostconfiguredcontainer_integration_test.go │ ├── types │ │ └── types.go │ ├── containersstate.go │ └── container_integration_test.go └── kubelet │ └── pool_test.go ├── cmd └── flexkube │ └── main.go ├── integration └── Dockerfile ├── libvirt ├── versions.tf ├── templates │ └── ct_config.yaml.tmpl ├── locals.tf ├── variables.tf └── main.tf ├── internal ├── util │ ├── validate_test.go │ ├── validate.go │ ├── util.go │ └── util_test.go └── utiltest │ ├── x509_test.go │ └── x509.go ├── Dockerfile ├── .gitignore ├── .goreleaser.yml ├── .semgrep.yml ├── docs └── RELEASING.md ├── Vagrantfile ├── .golangci.yml ├── .github └── workflows │ └── ci.yml └── README.md /.semgrepignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /local-testing/templates: -------------------------------------------------------------------------------- 1 | ../e2e/templates/ -------------------------------------------------------------------------------- /local-testing/main_test.go: -------------------------------------------------------------------------------- 1 | ../e2e/main_test.go -------------------------------------------------------------------------------- /local-testing/.gitignore: -------------------------------------------------------------------------------- 1 | values 2 | resources 3 | variables.auto.tfvars 4 | terraform-provider-flexkube 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .terraform 3 | bin 4 | e2e 5 | integration 6 | examples 7 | libvirt 8 | local-testing 9 | README.md 10 | LICENSE 11 | Vagrantfile 12 | kubeconfig 13 | CHANGELOG.md 14 | -------------------------------------------------------------------------------- /e2e/templates/kube-proxy-values.yaml.tmpl: -------------------------------------------------------------------------------- 1 | apiServers: 2 | {{- with (index .APILoadBalancerPools "controllers") }} 3 | {{- range $item := .Servers }} 4 | - {{ $item }} 5 | {{- end }} 6 | {{- end }} 7 | podCIDR: 10.1.0.0/16 8 | -------------------------------------------------------------------------------- /e2e/generate-secrets.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | test -f /root/.ssh/id_rsa || ssh-keygen -t ed25519 -f /root/.ssh/id_rsa -q -N "" 3 | test -f /root/.ssh/password || tr -cd "[:alnum:]" < /dev/urandom | fold -w30 | head -n1 | tr -d '\n' > /root/.ssh/password 4 | -------------------------------------------------------------------------------- /pkg/kubernetes/client/patches.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | // patchStringValue specifies a patch operation for a string. 4 | type patchStringValue struct { 5 | Op string `json:"op"` 6 | Path string `json:"path"` 7 | Value string `json:"value"` 8 | } 9 | -------------------------------------------------------------------------------- /cmd/flexkube/main.go: -------------------------------------------------------------------------------- 1 | // Package main provides runnable flexkube CLI binary. 2 | package main 3 | 4 | import ( 5 | "os" 6 | 7 | "github.com/flexkube/libflexkube/cli/flexkube" 8 | ) 9 | 10 | func main() { 11 | os.Exit(flexkube.Run(os.Args)) 12 | } 13 | -------------------------------------------------------------------------------- /e2e/98-ssh-as-root-with-password: -------------------------------------------------------------------------------- 1 | #!/usr/bin/with-contenv bash 2 | 3 | sed -i '/^AllowTcpForwarding/c\AllowTcpForwarding yes' /etc/ssh/sshd_config 4 | sed -i '/^GatewayPorts/c\GatewayPorts clientspecified' /etc/ssh/sshd_config 5 | echo "PermitRootLogin yes" >> /etc/ssh/sshd_config 6 | -------------------------------------------------------------------------------- /e2e/Dockerfile.openssh-server: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/linuxserver/openssh-server 2 | 3 | COPY ./generate-secrets.sh /usr/local/bin/generate-secrets.sh 4 | COPY ./98-ssh-as-root-with-password /custom-cont-init.d/98-ssh-as-root-with-password 5 | 6 | RUN chown root:root /custom-cont-init.d/98-ssh-as-root-with-password 7 | -------------------------------------------------------------------------------- /integration/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23-alpine3.20 2 | 3 | # Install dependencies: 4 | # - docker for spawning further Docker containers 5 | # - curl for installing CI tools 6 | # - build-base for make and gcc, required for running unit tests 7 | # - bash and bash-completion for usable shell 8 | # - vim for modifying files 9 | RUN apk add -U docker curl build-base bash bash-completion vim 10 | -------------------------------------------------------------------------------- /libvirt/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13" 3 | 4 | required_providers { 5 | null = { 6 | source = "hashicorp/null" 7 | version = "3.0.0" 8 | } 9 | libvirt = { 10 | source = "dmacvicar/libvirt" 11 | version = "0.6.12" 12 | } 13 | ct = { 14 | source = "poseidon/ct" 15 | version = "0.7.0" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pkg/kubernetes/client/clientset_test.go: -------------------------------------------------------------------------------- 1 | package client_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 7 | ) 8 | 9 | func TestNewClientset(t *testing.T) { 10 | t.Parallel() 11 | 12 | kubeconfig := GetKubeconfig(t) 13 | 14 | if _, err := client.NewClientset([]byte(kubeconfig)); err != nil { 15 | t.Fatalf("Creating clientset should work, got: %v", err) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /internal/util/validate_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestValidateErrors(t *testing.T) { 9 | t.Parallel() 10 | 11 | err := func() error { 12 | errors := ValidateErrors{ 13 | fmt.Errorf("first error"), 14 | } 15 | 16 | errors = append(errors, fmt.Errorf("second error")) 17 | 18 | return errors.Return() 19 | } 20 | 21 | if err() == nil { 22 | t.Fatalf("Error shouldn't be nil") 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /e2e/templates/kubeconfig.tmpl: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Config 3 | clusters: 4 | - name: ${name}-cluster 5 | cluster: 6 | server: ${server} 7 | certificate-authority-data: ${ca_cert} 8 | users: 9 | - name: ${name}-user 10 | user: 11 | client-certificate-data: ${client_cert} 12 | client-key-data: ${client_key} 13 | current-context: ${name}-context 14 | contexts: 15 | - name: ${name}-context 16 | context: 17 | cluster: ${name}-cluster 18 | namespace: kube-system 19 | user: ${name}-user 20 | -------------------------------------------------------------------------------- /pkg/kubernetes/client/clientset.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "fmt" 5 | 6 | "k8s.io/client-go/kubernetes" 7 | ) 8 | 9 | // NewClientset returns Kubernetes clientset object from kubeconfig string. 10 | func NewClientset(data []byte) (*kubernetes.Clientset, error) { 11 | cg, err := NewGetter(data) 12 | if err != nil { 13 | return nil, fmt.Errorf("creating kubernetes client getter: %w", err) 14 | } 15 | 16 | rc, err := cg.ToRESTConfig() 17 | if err != nil { 18 | return nil, fmt.Errorf("creating rest config: %w", err) 19 | } 20 | 21 | return kubernetes.NewForConfig(rc) 22 | } 23 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23-alpine3.20 2 | 3 | # Enable go modules 4 | ENV GO111MODULE=on 5 | 6 | # Install dependencies 7 | RUN apk add curl git build-base 8 | 9 | # Copy Makefile first to install CI binaries etc. 10 | ADD ./Makefile /usr/src/libflexkube/ 11 | 12 | WORKDIR /usr/src/libflexkube 13 | 14 | RUN make install-ci BIN_PATH=/usr/local/bin 15 | 16 | # Copy go mod files first and install dependencies to cache this layer 17 | ADD ./go.mod ./go.sum /usr/src/libflexkube/ 18 | 19 | RUN make download 20 | 21 | # Add source code 22 | ADD . /usr/src/libflexkube 23 | 24 | # Build, test and lint 25 | RUN make all build-bin 26 | -------------------------------------------------------------------------------- /libvirt/templates/ct_config.yaml.tmpl: -------------------------------------------------------------------------------- 1 | storage: 2 | files: 3 | - path: /etc/hostname 4 | filesystem: root 5 | mode: 0420 6 | contents: 7 | inline: | 8 | ${hostname} 9 | systemd: 10 | units: 11 | - name: locksmithd.service 12 | mask: true 13 | - name: update-engine.service 14 | mask: true 15 | - name: docker.service 16 | enabled: true 17 | - name: systemd-timesyncd.service 18 | enabled: true 19 | - name: containerd.service 20 | enabled: true 21 | passwd: 22 | users: 23 | - name: core 24 | ssh_authorized_keys: 25 | %{ for key in core_public_keys ~} 26 | - ${key} 27 | %{ endfor ~} 28 | -------------------------------------------------------------------------------- /internal/util/validate.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | // ValidateErrors is a collection of errors, which can be used when 8 | // performing validation of structs to collect all possible errors 9 | // and return them in one batch. 10 | type ValidateErrors []error 11 | 12 | func (e ValidateErrors) Error() string { 13 | errors := []string{} 14 | 15 | for _, s := range e { 16 | errors = append(errors, s.Error()) 17 | } 18 | 19 | return strings.Join(errors, ", ") 20 | } 21 | 22 | // Return returns nil, if no errors has been added. 23 | func (e ValidateErrors) Return() error { 24 | if len(e) > 0 { 25 | return e 26 | } 27 | 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /e2e/templates/kubernetes-values.yaml.tmpl: -------------------------------------------------------------------------------- 1 | serviceAccountPrivateKey: | 2 | {{ .State.PKI.Kubernetes.ServiceAccountCertificate.PrivateKey | toString | trim | indent 2 }} 3 | kubernetesCAKey: | 4 | {{ .State.PKI.Kubernetes.CA.PrivateKey | toString | trim | indent 2 }} 5 | kubernetesCACertificate: | 6 | {{ .State.PKI.Kubernetes.CA.X509Certificate | toString | trim | indent 2 }} 7 | rootCACertificate: | 8 | {{ .State.PKI.RootCA.X509Certificate | toString | trim | indent 2 }} 9 | 10 | replicas: {{ len .Etcd.Members }} 11 | flexVolumePluginDir: /var/lib/kubelet/volumeplugins 12 | apiServers: 13 | {{- with (index .APILoadBalancerPools "controllers") }} 14 | {{- range $item := .Servers }} 15 | - {{ $item }} 16 | {{- end }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /e2e/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: "3.8" 3 | services: 4 | generate-secrets: 5 | build: 6 | context: . 7 | dockerfile: Dockerfile.openssh-server 8 | entrypoint: 9 | - /usr/local/bin/generate-secrets.sh 10 | volumes: 11 | - ./secrets:/root/.ssh 12 | openssh-server: 13 | build: 14 | context: . 15 | dockerfile: Dockerfile.openssh-server 16 | environment: 17 | - USER_NAME=core 18 | - PUID=0 19 | - PGID=0 20 | - PUBLIC_KEY_FILE=/root/.ssh/id_rsa.pub 21 | - PASSWORD_ACCESS=true 22 | - USER_PASSWORD_FILE=/root/.ssh/password 23 | volumes: 24 | - ./secrets:/root/.ssh 25 | - /tmp:/tmp 26 | - /var/run/docker.sock:/var/run/docker.sock 27 | ports: 28 | - 2222:2222 29 | -------------------------------------------------------------------------------- /pkg/kubernetes/client/getter_test.go: -------------------------------------------------------------------------------- 1 | package client_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 7 | ) 8 | 9 | func TestGetter(t *testing.T) { 10 | t.Parallel() 11 | 12 | kubeconfig := GetKubeconfig(t) 13 | 14 | getter, err := client.NewGetter([]byte(kubeconfig)) 15 | if err != nil { 16 | t.Fatalf("Creating getter should work, got: %v", err) 17 | } 18 | 19 | if _, err := getter.ToDiscoveryClient(); err != nil { 20 | t.Errorf("Turning getter into discovery client should work, got: %v", err) 21 | } 22 | 23 | if _, err := getter.ToRESTMapper(); err != nil { 24 | t.Errorf("Turning getter into REST mapper should work, got: %v", err) 25 | } 26 | 27 | if c := getter.ToRawKubeConfigLoader(); c == nil { 28 | t.Errorf("Turning getter into RawKubeConfigLoader should work") 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # All coverage files. 2 | coverage.txt 3 | c.out 4 | 5 | # All CLI config and state files, except examples. 6 | state.yaml 7 | config.yaml 8 | !examples/**/config.yaml 9 | 10 | # Terraform-related directories. 11 | terraform.tfstate* 12 | .terraform 13 | crash.log 14 | .terraform.tfstate.lock.info 15 | *.auto.tfvars 16 | .terraform.lock.hcl 17 | 18 | # Built binaries 19 | /bin 20 | /flexkube 21 | 22 | # Vagrant-related directories. 23 | .vagrant 24 | 25 | # Generated kubeconfig files. 26 | kubeconfig 27 | 28 | # Local make settings. 29 | .env 30 | 31 | # Conformance test artifacts. 32 | *.tar.gz 33 | 34 | # Downloaded Flatcar images 35 | flatcar_production_qemu_image.img 36 | flatcar_production_qemu_image.img.bz2 37 | 38 | # Local test configs 39 | test-config.yaml 40 | 41 | # e2e artifacts. 42 | /e2e/resources 43 | /e2e/values 44 | /e2e/secrets 45 | -------------------------------------------------------------------------------- /pkg/helm/release/client.go: -------------------------------------------------------------------------------- 1 | package release 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/flexkube/helm/v3/pkg/kube" 7 | "k8s.io/client-go/kubernetes" 8 | "k8s.io/kubectl/pkg/cmd/util" 9 | 10 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 11 | ) 12 | 13 | // TODO maybe we should return struct here? 14 | func newClients(kubeconfig string) (*client.Getter, *kube.Client, *kubernetes.Clientset, error) { 15 | clientGetter, err := client.NewGetter([]byte(kubeconfig)) 16 | if err != nil { 17 | return nil, nil, nil, fmt.Errorf("creating Kubernetes client getter: %w", err) 18 | } 19 | 20 | client := &kube.Client{ 21 | Factory: util.NewFactory(clientGetter), 22 | Log: func(_ string, _ ...interface{}) {}, 23 | } 24 | 25 | kc, err := client.Factory.KubernetesClientSet() 26 | if err != nil { 27 | return nil, nil, nil, fmt.Errorf("creating Kubernetes client: %w", err) 28 | } 29 | 30 | return clientGetter, client, kc, nil 31 | } 32 | -------------------------------------------------------------------------------- /e2e/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23-alpine3.20 2 | 3 | RUN apk add -U make bash bash-completion vim coreutils 4 | 5 | RUN wget https://get.helm.sh/helm-v3.15.1-linux-amd64.tar.gz -O- | tar -xzvf - -C /usr/local/bin --strip-components=1 linux-amd64/helm 6 | 7 | RUN wget -O /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(wget -q -O- https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ 8 | chmod +x /usr/local/bin/kubectl && \ 9 | echo -e 'source /usr/share/bash-completion/bash_completion\nsource <(kubectl completion bash)' >> $HOME/.bashrc 10 | 11 | RUN VERSION=0.57.1 OS=linux && \ 12 | wget "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_amd64.tar.gz" -O- | \ 13 | tar -xzf - -C /usr/local/bin 14 | 15 | RUN helm repo add flexkube https://flexkube.github.io/charts/ 16 | 17 | ENV KUBECONFIG=/root/libflexkube/e2e/kubeconfig 18 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | builds: 2 | - id: flexkube 3 | env: 4 | - CGO_ENABLED=0 5 | ldflags: 6 | - -extldflags '-static' 7 | - -s 8 | - -w 9 | flags: 10 | - -trimpath 11 | goarch: 12 | - amd64 13 | goos: 14 | - linux 15 | - darwin 16 | main: ./cmd/flexkube 17 | binary: flexkube 18 | 19 | project_name: flexkube 20 | 21 | changelog: 22 | skip: true 23 | 24 | release: 25 | draft: true 26 | prerelease: true 27 | github: 28 | owner: flexkube 29 | name: libflexkube 30 | 31 | # Only add binaries to the archive files. 32 | archives: 33 | - id: flexkube 34 | builds: 35 | - flexkube 36 | name_template: "{{ .Binary }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}" 37 | files: 38 | - none* 39 | 40 | signs: 41 | - artifacts: all 42 | args: ["-u", "{{ .Env.GPG_FINGERPRINT }}", "--output", "${signature}", "--detach-sign", "${artifact}"] 43 | 44 | source: 45 | enabled: true 46 | -------------------------------------------------------------------------------- /pkg/etcd/fake.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | 6 | clientv3 "go.etcd.io/etcd/client/v3" 7 | ) 8 | 9 | type fakeClient struct { 10 | memberListF func(context context.Context) (*clientv3.MemberListResponse, error) 11 | memberAddF func(context context.Context, peerURLs []string) (*clientv3.MemberAddResponse, error) 12 | memberRemoveF func(context context.Context, id uint64) (*clientv3.MemberRemoveResponse, error) 13 | } 14 | 15 | func (f *fakeClient) MemberList(context context.Context) (*clientv3.MemberListResponse, error) { 16 | return f.memberListF(context) 17 | } 18 | 19 | func (f *fakeClient) MemberAdd(context context.Context, peerURLs []string) (*clientv3.MemberAddResponse, error) { 20 | return f.memberAddF(context, peerURLs) 21 | } 22 | 23 | func (f *fakeClient) MemberRemove(context context.Context, id uint64) (*clientv3.MemberRemoveResponse, error) { 24 | return f.memberRemoveF(context, id) 25 | } 26 | 27 | func (f *fakeClient) Close() error { 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /libvirt/locals.tf: -------------------------------------------------------------------------------- 1 | resource "null_resource" "controllers" { 2 | count = var.controllers_count 3 | 4 | triggers = { 5 | name = format("controller%02d", count.index + 1) 6 | ip = cidrhost(var.nodes_cidr, count.index + 2) 7 | cidr = cidrsubnet(var.pod_cidr, 8, count.index + 2) 8 | } 9 | } 10 | 11 | locals { 12 | controller_ips = null_resource.controllers.*.triggers.ip 13 | controller_names = null_resource.controllers.*.triggers.name 14 | controller_cidrs = null_resource.controllers.*.triggers.cidr 15 | 16 | first_controller_ip = local.controller_ips[0] 17 | 18 | worker_ips = null_resource.workers.*.triggers.ip 19 | worker_cidrs = null_resource.workers.*.triggers.cidr 20 | worker_names = null_resource.workers.*.triggers.name 21 | } 22 | 23 | resource "null_resource" "workers" { 24 | count = var.workers_count 25 | 26 | triggers = { 27 | name = format("worker%02d", count.index + 1) 28 | ip = cidrhost(var.nodes_cidr, count.index + 2 + 10) 29 | cidr = cidrsubnet(var.pod_cidr, 8, count.index + 2 + 10) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /pkg/pki/privatekey.go: -------------------------------------------------------------------------------- 1 | package pki 2 | 3 | import ( 4 | "crypto/x509" 5 | "encoding/pem" 6 | "fmt" 7 | ) 8 | 9 | // ValidatePrivateKey validates given private key in PEM format. 10 | // If decoding or parsing fails, error is returned. 11 | func ValidatePrivateKey(key string) error { 12 | der, _ := pem.Decode([]byte(key)) 13 | if der == nil { 14 | return fmt.Errorf("decoding PEM format") 15 | } 16 | 17 | if err := parsePrivateKey(der.Bytes); err != nil { 18 | return fmt.Errorf("parsing private key: %w", err) 19 | } 20 | 21 | return nil 22 | } 23 | 24 | // parsePrivateKey tries to parse various private key types and 25 | // returns error if none of them works. 26 | func parsePrivateKey(rawPrivateKey []byte) error { 27 | if _, err := x509.ParsePKCS8PrivateKey(rawPrivateKey); err == nil { 28 | return nil 29 | } 30 | 31 | if _, err := x509.ParsePKCS1PrivateKey(rawPrivateKey); err == nil { 32 | return nil 33 | } 34 | 35 | if _, err := x509.ParseECPrivateKey(rawPrivateKey); err == nil { 36 | return nil 37 | } 38 | 39 | return fmt.Errorf("unsupported private key format, tried PKCS8, PKCS1 and EC formats") 40 | } 41 | -------------------------------------------------------------------------------- /pkg/helm/release/release_examples_test.go: -------------------------------------------------------------------------------- 1 | package release_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/flexkube/libflexkube/pkg/helm/release" 9 | ) 10 | 11 | // Creating helm release. 12 | func ExampleConfig_New() { 13 | config := &release.Config{ 14 | // Put content of your kubeconfig file here. 15 | Kubeconfig: "", 16 | 17 | // The namespace must be created upfront. 18 | Namespace: "kube-system", 19 | 20 | // Name of helm release. 21 | Name: "coredns", 22 | 23 | // Repositories must be added upfront as well. 24 | Chart: "stable/coredns", 25 | 26 | // Values passed to the release in YAML format. 27 | Values: `replicas: 1 28 | labels: 29 | foo: bar 30 | `, 31 | // Version of the chart to use. 32 | Version: "1.12.0", 33 | } 34 | 35 | exampleRelease, err := config.New() 36 | if err != nil { 37 | fmt.Fprintf(os.Stderr, "Creating release object failed: %v\n", err) 38 | 39 | return 40 | } 41 | 42 | if err := exampleRelease.Install(context.TODO()); err != nil { 43 | fmt.Fprintf(os.Stderr, "Installing release failed: %v\n", err) 44 | 45 | return 46 | } 47 | // Output: 48 | } 49 | -------------------------------------------------------------------------------- /pkg/pki/privatekey_test.go: -------------------------------------------------------------------------------- 1 | package pki_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/internal/utiltest" 7 | "github.com/flexkube/libflexkube/pkg/pki" 8 | ) 9 | 10 | func TestPrivateKeyParse(t *testing.T) { 11 | t.Parallel() 12 | 13 | cases := map[string]struct { 14 | key string 15 | err bool 16 | }{ 17 | "bad": { 18 | "doh", 19 | true, 20 | }, 21 | "badpem": { 22 | ` 23 | -----BEGIN CERTIFICATE----- 24 | aHR0cHM6Ly93d3cueW91dHViZS5jb20vd2F0Y2g/dj1kUXc0dzlXZ1hjUQo= 25 | -----END CERTIFICATE----- 26 | `, 27 | true, 28 | }, 29 | "pkcs1": { 30 | utiltest.GeneratePKCS1PrivateKey(t), 31 | false, 32 | }, 33 | "ec": { 34 | utiltest.GenerateECPrivateKey(t), 35 | false, 36 | }, 37 | "rsa": { 38 | utiltest.GenerateRSAPrivateKey(t), 39 | false, 40 | }, 41 | } 42 | 43 | for n, testCase := range cases { 44 | testCase := testCase 45 | 46 | t.Run(n, func(t *testing.T) { 47 | t.Parallel() 48 | 49 | err := pki.ValidatePrivateKey(testCase.key) 50 | if testCase.err && err == nil { 51 | t.Fatalf("Expected error and didn't get any.") 52 | } 53 | 54 | if !testCase.err && err != nil { 55 | t.Fatalf("Didn't expect error, got: %v", err) 56 | } 57 | }) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/host/transport/transport.go: -------------------------------------------------------------------------------- 1 | // Package transport provides interfaces for forwarding connections. 2 | package transport 3 | 4 | // Interface Transport should be a valid object, which is ready to open connection. 5 | type Interface interface { 6 | // Connect initializes the connection with transport method. For example, if transport method 7 | // requires initial authentication, it should happen at this point, so further forward errors 8 | // are more specific. 9 | Connect() (Connected, error) 10 | } 11 | 12 | // Connected interface describes universal way of communicating with remote hosts 13 | // using different transport protocols. 14 | type Connected interface { 15 | // ForwardUnixSocket forwards unix socket to local machine to make it available for the process. 16 | ForwardUnixSocket(remotePath string) (localPath string, err error) 17 | 18 | // ForwardTCP listens on random local port and forwards incoming connections to given remote address. 19 | ForwardTCP(remoteAddr string) (localAddr string, err error) 20 | } 21 | 22 | // Config describes how Transport interface should be created. 23 | type Config interface { 24 | // New returns new instance of Transport object. 25 | New() (Interface, error) 26 | 27 | // Validate should validate Transport configuration. 28 | Validate() error 29 | } 30 | -------------------------------------------------------------------------------- /pkg/defaults/defaults.go: -------------------------------------------------------------------------------- 1 | // Package defaults provides default values used across the library. 2 | package defaults 3 | 4 | const ( 5 | // EtcdImage points to a default Docker image, which will be used for running etcd. 6 | EtcdImage = "quay.io/coreos/etcd:v3.5.16" 7 | 8 | // KubeAPIServerImage points to a default Docker image, which will be used for 9 | // running kube-apiserver. 10 | KubeAPIServerImage = "registry.k8s.io/kube-apiserver:v1.31.0" 11 | 12 | // KubeControllerManagerImage points to a default Docker image, which will be used for 13 | // running kube-apiserver. 14 | KubeControllerManagerImage = "registry.k8s.io/kube-controller-manager:v1.31.0" 15 | 16 | // KubeSchedulerImage points to a default Docker image, which will be used for 17 | // running kube-apiserver. 18 | KubeSchedulerImage = "registry.k8s.io/kube-scheduler:v1.31.0" 19 | 20 | // KubeletImage points to a default Docker image, which will be used for 21 | // running kube-apiserver. 22 | KubeletImage = "quay.io/flexkube/kubelet:v1.31.0" 23 | 24 | // HAProxyImage is a default container image for APILoadBalancer. 25 | HAProxyImage = "haproxy:3.0.4-alpine" 26 | 27 | // DockerAPIVersion is a default API version used when talking to Docker runtime. 28 | DockerAPIVersion = "v1.38" 29 | 30 | // VolumePluginDir is a default flex volume plugin directory configured for kubelet 31 | // and kube-controller-manager. 32 | VolumePluginDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" 33 | ) 34 | -------------------------------------------------------------------------------- /.semgrep.yml: -------------------------------------------------------------------------------- 1 | rules: 2 | - id: bad-error-wrap-wording 3 | patterns: 4 | - pattern-regex: fmt\.Errorf\(".?([Ff]ailed|[]Uu]nable|[Cc]ould|([Ee]rror)) 5 | message: Word "fail", "failed", "unable", "could not" should not be used when wrapping errors. Use verb in continuous form instead! 6 | languages: [go] 7 | severity: ERROR 8 | - id: printed-messages-must-be-captialized 9 | patterns: 10 | - pattern-regex: (fmt.Print(ln|f)|\bt\.(Error|Fatal|Log)(|f))\("[a-z] 11 | message: Printed messages must be capitalized. 12 | languages: [go] 13 | severity: ERROR 14 | - id: strings-must-be-quoted-using-q 15 | patterns: 16 | - pattern-regex: "'%s'" 17 | message: When formatting, strings must be quoted using %q instead of e.g. '%s'. 18 | languages: [go] 19 | severity: ERROR 20 | - id: error-wrapping-must-be-preceded-by-space 21 | patterns: 22 | - pattern-regex: '[^ ]%w' 23 | message: When wrapping errors, %w must be preceded by space. 24 | languages: [go] 25 | severity: ERROR 26 | - id: files-must-not-have-trailing-whitespace 27 | patterns: 28 | - pattern-regex: '[[:blank:]]$' 29 | message: Files must not have any trailing whitespace. 30 | languages: [generic] 31 | severity: ERROR 32 | - id: files-must-not-have-trailing-newlines 33 | patterns: 34 | - pattern-regex: '\n\n\Z' 35 | message: Files must not have any trailing newlines. 36 | languages: [generic] 37 | severity: ERROR 38 | - id: all-lines-must-end-with-newline 39 | patterns: 40 | - pattern-regex: '\S\z' 41 | message: All lines must end with newline. 42 | languages: [generic] 43 | severity: ERROR 44 | -------------------------------------------------------------------------------- /pkg/controlplane/validate_test.go: -------------------------------------------------------------------------------- 1 | package controlplane 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/internal/utiltest" 7 | "github.com/flexkube/libflexkube/pkg/host" 8 | "github.com/flexkube/libflexkube/pkg/host/transport/direct" 9 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 10 | "github.com/flexkube/libflexkube/pkg/types" 11 | ) 12 | 13 | func validValidator(t *testing.T) validator { 14 | t.Helper() 15 | 16 | pki := utiltest.GeneratePKI(t) 17 | 18 | hostConfig := &host.Host{ 19 | DirectConfig: &direct.Config{}, 20 | } 21 | 22 | common := &Common{ 23 | KubernetesCACertificate: types.Certificate(pki.Certificate), 24 | FrontProxyCACertificate: types.Certificate(pki.Certificate), 25 | } 26 | 27 | kubeconfig := client.Config{ 28 | Server: "localhost", 29 | CACertificate: types.Certificate(pki.Certificate), 30 | ClientCertificate: types.Certificate(pki.Certificate), 31 | ClientKey: types.PrivateKey(pki.PrivateKey), 32 | } 33 | 34 | return validator{ 35 | Common: common, 36 | Kubeconfig: kubeconfig, 37 | Host: hostConfig, 38 | } 39 | } 40 | 41 | func TestValidate(t *testing.T) { 42 | t.Parallel() 43 | 44 | v := validValidator(t) 45 | 46 | if err := v.validate(true); err != nil { 47 | t.Fatalf("Validating valid object should succeed, got: %v", err) 48 | } 49 | } 50 | 51 | func TestValidateMarshalFail(t *testing.T) { 52 | t.Parallel() 53 | 54 | testValidator := validValidator(t) 55 | 56 | testValidator.YAML = map[string]interface{}{ 57 | "foo": make(chan int), 58 | } 59 | 60 | if err := testValidator.validate(true); err == nil { 61 | t.Fatalf("Validating unmarshalable struct should fail") 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /pkg/types/certificate.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "crypto/x509" 5 | "encoding/pem" 6 | "fmt" 7 | "strconv" 8 | 9 | "github.com/flexkube/libflexkube/internal/util" 10 | ) 11 | 12 | // Certificate is a wrapper on string type, which parses it's content 13 | // as X.509 certificate while unmarshaling. This allows to validate the 14 | // data during unmarshaling process. 15 | // 16 | // This type should not be used, as it does not allow to produce 17 | // meaningful error to the user. 18 | type Certificate string 19 | 20 | // UnmarshalJSON implements encoding/json.Unmarshaler interface and tries 21 | // to parse obtained data as PEM encoded X.509 certificate. 22 | func (c *Certificate) UnmarshalJSON(data []byte) error { 23 | unquoted, err := strconv.Unquote(string(data)) 24 | if err != nil { 25 | return fmt.Errorf("unquoting string: %w", err) 26 | } 27 | 28 | der, _ := pem.Decode([]byte(unquoted)) 29 | if der == nil { 30 | return fmt.Errorf("decoding PEM format") 31 | } 32 | 33 | if _, err := x509.ParseCertificate(der.Bytes); err != nil { 34 | return fmt.Errorf("parsing certificate: %w", err) 35 | } 36 | 37 | *c = Certificate(unquoted) 38 | 39 | return nil 40 | } 41 | 42 | // Pick returns first non-empty certificate from given list, including 43 | // receiver certificate. 44 | // 45 | // This method is a helper, which allows to select the certificate to use 46 | // from hierarchical configuration. 47 | func (c *Certificate) Pick(values ...Certificate) Certificate { 48 | if c == nil || *c == "" { 49 | ce := Certificate("") 50 | c = &ce 51 | } 52 | 53 | cs := []string{string(*c)} 54 | for _, v := range values { 55 | cs = append(cs, string(v)) 56 | } 57 | 58 | return Certificate(util.PickString(cs...)) 59 | } 60 | -------------------------------------------------------------------------------- /e2e/templates/kube-apiserver-values.yaml.tmpl: -------------------------------------------------------------------------------- 1 | serverKey: | 2 | {{ .State.PKI.Kubernetes.KubeAPIServer.ServerCertificate.PrivateKey | toString | trim | indent 2 }} 3 | serverCertificate: | 4 | {{ .State.PKI.Kubernetes.KubeAPIServer.ServerCertificate.X509Certificate | toString | trim | indent 2 }} 5 | serviceAccountPrivateKey: | 6 | {{ .State.PKI.Kubernetes.ServiceAccountCertificate.PrivateKey | toString | trim | indent 2 }} 7 | caCertificate: | 8 | {{ .State.PKI.Kubernetes.CA.X509Certificate | toString | trim | indent 2 }} 9 | frontProxyClientKey: | 10 | {{ .State.PKI.Kubernetes.KubeAPIServer.FrontProxyClientCertificate.PrivateKey | toString | trim | indent 2 }} 11 | frontProxyClientCertificate: | 12 | {{ .State.PKI.Kubernetes.KubeAPIServer.FrontProxyClientCertificate.X509Certificate | toString | trim | indent 2 }} 13 | frontProxyCACertificate: | 14 | {{ .State.PKI.Kubernetes.FrontProxyCA.X509Certificate | toString | trim | indent 2 }} 15 | kubeletClientCertificate: | 16 | {{ .State.PKI.Kubernetes.KubeAPIServer.KubeletCertificate.X509Certificate | toString | trim | indent 2 }} 17 | kubeletClientKey: | 18 | {{ .State.PKI.Kubernetes.KubeAPIServer.KubeletCertificate.PrivateKey | toString | trim | indent 2 }} 19 | etcdCACertificate: | 20 | {{ .State.PKI.Etcd.CA.X509Certificate | toString | trim | indent 2 }} 21 | etcdClientCertificate: | 22 | {{ with (index .State.PKI.Etcd.ClientCertificates "kube-apiserver") }}{{ .X509Certificate | toString | trim | indent 2 }}{{ end }} 23 | etcdClientKey: | 24 | {{ with (index .State.PKI.Etcd.ClientCertificates "kube-apiserver") }}{{ .PrivateKey | toString | trim | indent 2 }}{{ end }} 25 | etcdServers: 26 | {{- range $item := .Controlplane.KubeAPIServer.EtcdServers }} 27 | - {{ $item }} 28 | {{- end }} 29 | serviceCIDR: {{ .Controlplane.KubeAPIServer.ServiceCIDR }} 30 | replicas: {{ len .Etcd.Members }} 31 | -------------------------------------------------------------------------------- /libvirt/variables.tf: -------------------------------------------------------------------------------- 1 | variable "ssh_private_key_path" { 2 | default = "/root/.ssh/id_rsa" 3 | } 4 | 5 | variable "controllers_count" { 6 | default = 1 7 | } 8 | 9 | variable "workers_count" { 10 | default = 0 11 | } 12 | 13 | variable "nodes_cidr" { 14 | default = "192.168.50.0/24" 15 | } 16 | 17 | variable "pod_cidr" { 18 | default = "10.1.0.0/16" 19 | } 20 | 21 | variable "network_plugin" { 22 | default = "calico" 23 | } 24 | 25 | variable "node_ssh_port" { 26 | default = 22 27 | } 28 | 29 | variable "kube_apiserver_helm_chart_source" { 30 | default = "flexkube/kube-apiserver" 31 | } 32 | 33 | variable "kubernetes_helm_chart_source" { 34 | default = "flexkube/kubernetes" 35 | } 36 | 37 | variable "kube_proxy_helm_chart_source" { 38 | default = "flexkube/kube-proxy" 39 | } 40 | 41 | variable "tls_bootstrapping_helm_chart_source" { 42 | default = "flexkube/tls-bootstrapping" 43 | } 44 | 45 | variable "kubelet_rubber_stamp_helm_chart_source" { 46 | default = "flexkube/kubelet-rubber-stamp" 47 | } 48 | 49 | variable "calico_helm_chart_source" { 50 | default = "flexkube/calico" 51 | } 52 | 53 | variable "kube_apiserver_helm_chart_version" { 54 | default = "0.2.0" 55 | } 56 | 57 | variable "kubernetes_helm_chart_version" { 58 | default = "0.3.2" 59 | } 60 | 61 | variable "kube_proxy_helm_chart_version" { 62 | default = "0.2.3" 63 | } 64 | 65 | variable "tls_bootstrapping_helm_chart_version" { 66 | default = "0.1.1" 67 | } 68 | 69 | variable "coredns_chart_version" { 70 | default = "1.13.3" 71 | } 72 | 73 | variable "metrics_server_chart_version" { 74 | default = "2.11.1" 75 | } 76 | 77 | variable "kubelet_rubber_stamp_helm_chart_version" { 78 | default = "0.1.4" 79 | } 80 | 81 | variable "calico_helm_chart_version" { 82 | default = "0.2.4" 83 | } 84 | 85 | variable "flatcar_channel" { 86 | default = "edge" 87 | } 88 | -------------------------------------------------------------------------------- /pkg/host/transport/direct/direct.go: -------------------------------------------------------------------------------- 1 | // Package direct is a transport.Interface implementation, which simply 2 | // forwards given addresses "as is", without any modifications. 3 | package direct 4 | 5 | import ( 6 | "fmt" 7 | "net" 8 | 9 | "github.com/flexkube/libflexkube/pkg/host/transport" 10 | ) 11 | 12 | // Config represents host configuration for direct communication. 13 | // 14 | // Using this struct will use local network and local filesystem. 15 | type Config struct { 16 | // Dummy field is only user for testing. 17 | Dummy string `json:"-"` 18 | } 19 | 20 | // direct is a initialized struct, which satisfies Transport interface. 21 | type direct struct{} 22 | 23 | // New may in the future validate direct configuration. 24 | func (c *Config) New() (transport.Interface, error) { 25 | return &direct{}, nil 26 | } 27 | 28 | // Validate validates Config struct. 29 | // 30 | // Currently it's not doing anything, but it's here for compatibility purposes 31 | // with other types. In the future some validation rules may be added. 32 | func (c *Config) Validate() error { 33 | return nil 34 | } 35 | 36 | // ForwardUnixSocket returns forwarded UNIX socket. 37 | // 38 | // Given that direct operates on local filesystem, it simply returns given path. 39 | // 40 | // TODO perhaps try to connect to given socket to see if it exists, we have permissions 41 | // etc to fail early? 42 | func (d *direct) ForwardUnixSocket(path string) (string, error) { 43 | return path, nil 44 | } 45 | 46 | // Connect implements Transport interface. 47 | func (d *direct) Connect() (transport.Connected, error) { 48 | return d, nil 49 | } 50 | 51 | func (d *direct) ForwardTCP(address string) (string, error) { 52 | if _, _, err := net.SplitHostPort(address); err != nil { 53 | return "", fmt.Errorf("validating address %q: %w", address, err) 54 | } 55 | 56 | return address, nil 57 | } 58 | -------------------------------------------------------------------------------- /pkg/host/transport/ssh/defaults.go: -------------------------------------------------------------------------------- 1 | package ssh 2 | 3 | import ( 4 | "github.com/flexkube/libflexkube/internal/util" 5 | ) 6 | 7 | const ( 8 | // User is a default user used for SSH connections. 9 | User = "root" 10 | 11 | // ConnectionTimeout is a default time SSH will wait while connecting to unreachable server. 12 | ConnectionTimeout = "30s" 13 | 14 | // RetryTimeout is a default time after we give up connecting to unreachable server. 15 | RetryTimeout = "60s" 16 | 17 | // RetryInterval is a default time how long we wait between SSH connection attempts. 18 | RetryInterval = "1s" 19 | 20 | // Port is a default port used for SSH connections. 21 | Port = 22 22 | ) 23 | 24 | // BuildConfig takes destination SSH configuration, struct with default values provided by the user 25 | // and merges it together with global SSH default values. 26 | func BuildConfig(sshConfig, defaults *Config) *Config { 27 | if sshConfig == nil { 28 | sshConfig = &Config{} 29 | } 30 | 31 | if defaults == nil { 32 | defaults = &Config{} 33 | } 34 | 35 | sshConfig.PrivateKey = util.PickString(sshConfig.PrivateKey, defaults.PrivateKey) 36 | 37 | sshConfig.User = util.PickString(sshConfig.User, defaults.User, User) 38 | 39 | sshConfig.ConnectionTimeout = util.PickString( 40 | sshConfig.ConnectionTimeout, 41 | defaults.ConnectionTimeout, 42 | ConnectionTimeout, 43 | ) 44 | 45 | sshConfig.RetryTimeout = util.PickString(sshConfig.RetryTimeout, defaults.RetryTimeout, RetryTimeout) 46 | 47 | sshConfig.RetryInterval = util.PickString(sshConfig.RetryInterval, defaults.RetryInterval, RetryInterval) 48 | 49 | sshConfig.Port = util.PickInt(sshConfig.Port, defaults.Port, Port) 50 | 51 | sshConfig.Address = util.PickString(sshConfig.Address, defaults.Address) 52 | 53 | sshConfig.Password = util.PickString(sshConfig.Password, defaults.Password) 54 | 55 | return sshConfig 56 | } 57 | -------------------------------------------------------------------------------- /pkg/types/certificate_test.go: -------------------------------------------------------------------------------- 1 | package types_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "sigs.k8s.io/yaml" 9 | 10 | "github.com/flexkube/libflexkube/internal/util" 11 | "github.com/flexkube/libflexkube/internal/utiltest" 12 | "github.com/flexkube/libflexkube/pkg/types" 13 | ) 14 | 15 | func TestCertificateParse(t *testing.T) { 16 | t.Parallel() 17 | 18 | type Foo struct { 19 | Bar types.Certificate `json:"bar"` 20 | } 21 | 22 | cases := map[string]struct { 23 | YAML string 24 | Error bool 25 | }{ 26 | "bad": { 27 | "bar: doh", 28 | true, 29 | }, 30 | "good": { 31 | fmt.Sprintf("bar: |\n%s", util.Indent(strings.TrimSpace(utiltest.GenerateX509Certificate(t)), " ")), 32 | false, 33 | }, 34 | } 35 | 36 | for n, testCase := range cases { 37 | testCase := testCase 38 | 39 | t.Run(n, func(t *testing.T) { 40 | t.Parallel() 41 | 42 | bar := &Foo{} 43 | 44 | err := yaml.Unmarshal([]byte(testCase.YAML), bar) 45 | if testCase.Error && err == nil { 46 | t.Fatalf("Expected error and didn't get any.") 47 | } 48 | 49 | if !testCase.Error && err != nil { 50 | t.Fatalf("Didn't expect error, got: %v", err) 51 | } 52 | 53 | if err == nil && bar.Bar == "" { 54 | t.Fatalf("Didn't get any error, but field is empty") 55 | } 56 | 57 | if err != nil && bar.Bar != "" { 58 | t.Fatalf("Got error and still got some content") 59 | } 60 | }) 61 | } 62 | } 63 | 64 | func TestCertificatePickNil(t *testing.T) { 65 | t.Parallel() 66 | 67 | var c types.Certificate 68 | 69 | if c.Pick(types.Certificate("bar"), types.Certificate("baz")) != "bar" { 70 | t.Fatalf("First non empty certificate should be picked") 71 | } 72 | } 73 | 74 | func TestCertificatePick(t *testing.T) { 75 | t.Parallel() 76 | 77 | d := types.Certificate("foo") 78 | 79 | if d.Pick(types.Certificate("baz")) != "foo" { 80 | t.Fatalf("First non empty certificate should be picked") 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /docs/RELEASING.md: -------------------------------------------------------------------------------- 1 | # libflexkube release process 2 | 3 | This document briefly describes the process of releasing new version of libflexkube. 4 | 5 | ## Before the release 6 | 7 | Before creating a Git tag and a GitHub release, following tasks should be performed: 8 | 9 | - Changelog for the new version should be added to CHANGELOG.md file. 10 | - Changelog link should be added at the bottom of CHANGELOG.md file. 11 | - `Version` constant in `cli/flexkube/cli.go` file should be changed to version which will be released and change should be committed. This commit will be later on tagged while releasing, so it should be done as last action before the release. 12 | - `Version` constant in `cli/flexkube/cli.go` file should be changed to next version with `-unreleased` suffix and change should be committed. This commit will be the first commit of the next release. 13 | - Conformance tests (e.g. `make vagrant-conformance`) should be performed on the release commit before creating an actual release to ensure the release is working properly. 14 | - Before creating a Pull Request, run `goreleaser --skip-publish` to ensure that the release will build for all desired platforms. 15 | - Pull Request with described changes should be created and merged. 16 | 17 | ## Creating the release 18 | 19 | To create new release, following tasks should be performed: 20 | 21 | - Tag new release on desired commit with CLI version changed, using example command: 22 | 23 | ```sh 24 | git tag -a v0.4.7 -s -m "Release v0.4.7" 25 | ``` 26 | 27 | - Push tag to GitHub: 28 | 29 | ```sh 30 | git push upstream v0.4.7 31 | ``` 32 | 33 | - Run `goreleaser` to create a GitHub Release: 34 | 35 | ```sh 36 | GITHUB_TOKEN=githubtoken goreleaser release --release-notes <(go run github.com/rcmachado/changelog@0.7.0 show 0.4.7) 37 | ``` 38 | 39 | - Go to newly create [GitHub release](https://github.com/flexkube/libflexkube/releases/tag/v0.4.7), verify that the changelog and artifacts looks correct and publish it. 40 | -------------------------------------------------------------------------------- /internal/utiltest/x509_test.go: -------------------------------------------------------------------------------- 1 | package utiltest 2 | 3 | import ( 4 | "crypto/x509" 5 | "encoding/pem" 6 | "testing" 7 | ) 8 | 9 | // GenerateX509Certificate() tests. 10 | func TestGenerateX509Certificate(t *testing.T) { 11 | t.Parallel() 12 | 13 | if a := GenerateX509Certificate(t); a == "" { 14 | t.Fatalf("Generating X509 certificate should not return empty string") 15 | } 16 | } 17 | 18 | // GenerateRSAPrivateKey() tests. 19 | func TestGenerateRSAPrivateKey(t *testing.T) { 20 | t.Parallel() 21 | 22 | if a := GenerateRSAPrivateKey(t); a == "" { 23 | t.Fatalf("Generating RSA private key should not return empty string") 24 | } 25 | } 26 | 27 | // GeneratePKCS1PrivateKey tests. 28 | func Test_GeneratePKCS1PrivateKey_returns_PEM_encoded_RSA_private_key(t *testing.T) { 29 | t.Parallel() 30 | 31 | pemEncodedPrivateKey := GeneratePKCS1PrivateKey(t) 32 | 33 | derPrivateKey, _ := pem.Decode([]byte(pemEncodedPrivateKey)) 34 | if derPrivateKey == nil { 35 | t.Fatalf("Returned key is not PEM encoded:\n%s", pemEncodedPrivateKey) 36 | } 37 | 38 | if _, err := x509.ParsePKCS1PrivateKey(derPrivateKey.Bytes); err != nil { 39 | t.Fatalf("Returned key is not PKCS1 private key") 40 | } 41 | } 42 | 43 | // GenerateECPrivateKey tests. 44 | func Test_GenerateECPrivateKey_returns_PEM_encoded_EC_private_key(t *testing.T) { 45 | t.Parallel() 46 | 47 | pemEncodedPrivateKey := GenerateECPrivateKey(t) 48 | 49 | derPrivateKey, _ := pem.Decode([]byte(pemEncodedPrivateKey)) 50 | if derPrivateKey == nil { 51 | t.Fatalf("Returned key is not PEM encoded:\n%s", pemEncodedPrivateKey) 52 | } 53 | 54 | if _, err := x509.ParseECPrivateKey(derPrivateKey.Bytes); err != nil { 55 | t.Fatalf("Returned key is not EC private key") 56 | } 57 | } 58 | 59 | // GeneratePKI() tests. 60 | func TestGeneratePKI(t *testing.T) { 61 | t.Parallel() 62 | 63 | testPKI := GeneratePKI(t) 64 | 65 | if testPKI.Certificate == "" { 66 | t.Errorf("PKI shouldn't have empty certificate field") 67 | } 68 | 69 | if testPKI.PrivateKey == "" { 70 | t.Errorf("PKI shouldn't have empty private key") 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /pkg/helm/release/release_internal_test.go: -------------------------------------------------------------------------------- 1 | package release 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestRetryOnEtcdErrorRetry(t *testing.T) { 9 | t.Parallel() 10 | 11 | calls := 0 12 | 13 | if err := retryOnEtcdError(func() error { 14 | calls++ 15 | 16 | return fmt.Errorf("etcdserver: foo") 17 | }); err == nil { 18 | t.Errorf("Retry should return error if all attempts failed") 19 | } 20 | 21 | if calls < 2 { 22 | t.Errorf("Function should be at least called twice on etcd error") 23 | } 24 | } 25 | 26 | func TestRetryOnEtcdErrorDifferentError(t *testing.T) { 27 | t.Parallel() 28 | 29 | calls := 0 30 | 31 | expectedError := fmt.Errorf("expected error") 32 | 33 | err := retryOnEtcdError(func() error { 34 | calls++ 35 | 36 | return expectedError 37 | }) 38 | if err == nil { 39 | t.Errorf("Retry should return error if all attempts failed") 40 | } 41 | 42 | if calls != 1 { 43 | t.Errorf("Function should be called only once if the error returned is not etcd error") 44 | } 45 | 46 | if err != expectedError { //nolint:errorlint // Helm errors cannot be unwrapped. 47 | t.Fatalf("Retrying did not retain original error, got: %v", err) 48 | } 49 | } 50 | 51 | func TestRetryOnEtcdErrorNoError(t *testing.T) { 52 | t.Parallel() 53 | 54 | calls := 0 55 | 56 | if err := retryOnEtcdError(func() error { 57 | calls++ 58 | 59 | return nil 60 | }); err != nil { 61 | t.Errorf("Retry should not return error, got: %v", err) 62 | } 63 | 64 | if calls != 1 { 65 | t.Errorf("Function should be called only once if no error is returned") 66 | } 67 | } 68 | 69 | func TestRetryOnEtcdErrorTranscientError(t *testing.T) { 70 | t.Parallel() 71 | 72 | calls := 0 73 | 74 | if err := retryOnEtcdError(func() error { 75 | calls++ 76 | 77 | if calls == 1 { 78 | return fmt.Errorf("etcdserver: foo") 79 | } 80 | 81 | return nil 82 | }); err != nil { 83 | t.Errorf("Retry should retry and not return error, got: %v", err) 84 | } 85 | 86 | if calls != 2 { 87 | t.Errorf("Function should return when no error is returned") 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /pkg/apiloadbalancer/api-loadbalancer_test.go: -------------------------------------------------------------------------------- 1 | package apiloadbalancer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/pkg/host" 7 | "github.com/flexkube/libflexkube/pkg/host/transport/direct" 8 | ) 9 | 10 | func TestToHostConfiguredContainer(t *testing.T) { 11 | t.Parallel() 12 | 13 | testLB := &APILoadBalancer{ 14 | Host: host.Host{ 15 | DirectConfig: &direct.Config{}, 16 | }, 17 | Servers: []string{"localhost:9090"}, 18 | BindAddress: "0.0.0.0:6434", 19 | } 20 | 21 | k, err := testLB.New() 22 | if err != nil { 23 | t.Fatalf("Creating new api loadbalancer should succeed, got: %v", err) 24 | } 25 | 26 | hcc, err := k.ToHostConfiguredContainer() 27 | if err != nil { 28 | t.Fatalf("Generating HostConfiguredContainer should work, got: %v", err) 29 | } 30 | 31 | if _, err := hcc.New(); err != nil { 32 | t.Fatalf("Should produce valid HostConfiguredContainer, got: %v", err) 33 | } 34 | 35 | if hcc.Container.Config.User == "" { 36 | t.Fatalf("HostConfiguredContainer should have user set") 37 | } 38 | } 39 | 40 | // Validate() tests. 41 | func TestValidateRequireServers(t *testing.T) { 42 | t.Parallel() 43 | 44 | testLB := &APILoadBalancer{ 45 | BindAddress: "0.0.0.0:6434", 46 | Host: host.Host{ 47 | DirectConfig: &direct.Config{}, 48 | }, 49 | } 50 | 51 | if err := testLB.Validate(); err == nil { 52 | t.Fatalf("Validate should require at least one server to be defined") 53 | } 54 | } 55 | 56 | func TestValidateRequireBindAddress(t *testing.T) { 57 | t.Parallel() 58 | 59 | testLB := &APILoadBalancer{ 60 | Servers: []string{"foo"}, 61 | Host: host.Host{ 62 | DirectConfig: &direct.Config{}, 63 | }, 64 | } 65 | 66 | if err := testLB.Validate(); err == nil { 67 | t.Fatalf("Validate should require at least one server to be defined") 68 | } 69 | } 70 | 71 | // New() tests. 72 | func TestNewValidate(t *testing.T) { 73 | t.Parallel() 74 | 75 | testLB := &APILoadBalancer{ 76 | Host: host.Host{ 77 | DirectConfig: &direct.Config{}, 78 | }, 79 | } 80 | 81 | if _, err := testLB.New(); err == nil { 82 | t.Fatalf("New should validate configuration before creating object") 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /pkg/container/runtime/runtime.go: -------------------------------------------------------------------------------- 1 | // Package runtime provides interfaces describing container runtimes 2 | // in generic way and their functionality. 3 | package runtime 4 | 5 | import ( 6 | "os" 7 | 8 | "github.com/flexkube/libflexkube/pkg/container/types" 9 | ) 10 | 11 | // Runtime interface describes universal way of managing containers 12 | // across different container runtimes. 13 | type Runtime interface { 14 | // Create creates container and returns it's unique identifier. 15 | Create(config *types.ContainerConfig) (string, error) 16 | 17 | // Delete removes the container. 18 | Delete(ID string) error 19 | 20 | // Start starts created container. 21 | Start(ID string) error 22 | 23 | // Status returns status of the container. 24 | Status(ID string) (types.ContainerStatus, error) 25 | 26 | // Stop takes unique identifier as a parameter and stops the container. 27 | Stop(ID string) error 28 | 29 | // Copy allows to copy TAR archive into the container. 30 | // 31 | // Docker currently does not allow to copy multiple files over https://github.com/moby/moby/issues/7710 32 | // It seems kubelet does https://github.com/kubernetes/kubernetes/pull/72641/files 33 | Copy(ID string, files []*types.File) error 34 | 35 | // Read allows to read file in TAR archive format from container. 36 | // 37 | // TODO check if we should return some information about read file 38 | Read(ID string, srcPath []string) ([]*types.File, error) 39 | 40 | // Stat returns os.FileMode for requested files from inside the container. 41 | Stat(ID string, paths []string) (map[string]os.FileMode, error) 42 | } 43 | 44 | // Config defines interface for runtime configuration. Since some feature are generic to runtime, 45 | // this interface make sure that other parts of the system are compatible with it. 46 | type Config interface { 47 | // GetAddress should return the URL, which will be used for talking to container runtime. 48 | GetAddress() string 49 | 50 | // SetAddress allows to override, which URL will be used when talking to container runtime. 51 | SetAddress(newAddress string) 52 | 53 | // New validates container runtime and returns object, which can be used to create containers etc. 54 | New() (Runtime, error) 55 | } 56 | -------------------------------------------------------------------------------- /pkg/kubernetes/client/getter.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "fmt" 5 | 6 | "k8s.io/apimachinery/pkg/api/meta" 7 | "k8s.io/client-go/discovery" 8 | "k8s.io/client-go/discovery/cached/memory" 9 | "k8s.io/client-go/rest" 10 | "k8s.io/client-go/restmapper" 11 | "k8s.io/client-go/tools/clientcmd" 12 | ) 13 | 14 | // Getter implements k8s.io/cli-runtime/pkg/genericclioptions.RESTClientGetter interface. 15 | type Getter struct { 16 | c clientcmd.ClientConfig 17 | } 18 | 19 | // ToRESTMapper is part of k8s.io/cli-runtime/pkg/genericclioptions.RESTClientGetter interface. 20 | func (c *Getter) ToRESTMapper() (meta.RESTMapper, error) { 21 | discoveryClient, err := c.ToDiscoveryClient() 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) 27 | expander := restmapper.NewShortcutExpander(mapper, discoveryClient) 28 | 29 | return expander, nil 30 | } 31 | 32 | // ToDiscoveryClient is part of k8s.io/cli-runtime/pkg/genericclioptions.RESTClientGetter interface. 33 | func (c *Getter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { 34 | cc, err := c.ToRESTConfig() 35 | if err != nil { 36 | return nil, fmt.Errorf("getting REST config: %w", err) 37 | } 38 | 39 | d, err := discovery.NewDiscoveryClientForConfig(cc) 40 | if err != nil { 41 | return nil, fmt.Errorf("creating discovery client: %w", err) 42 | } 43 | 44 | return memory.NewMemCacheClient(d), nil 45 | } 46 | 47 | // ToRawKubeConfigLoader is part of k8s.io/cli-runtime/pkg/genericclioptions.RESTClientGetter interface. 48 | func (c *Getter) ToRawKubeConfigLoader() clientcmd.ClientConfig { 49 | return c.c 50 | } 51 | 52 | // ToRESTConfig is part of k8s.io/cli-runtime/pkg/genericclioptions.RESTClientGetter interface. 53 | func (c *Getter) ToRESTConfig() (*rest.Config, error) { 54 | return c.c.ClientConfig() 55 | } 56 | 57 | // NewGetter takes content of kubeconfig file as an argument and returns implementation of 58 | // RESTClientGetter k8s interface. 59 | func NewGetter(data []byte) (*Getter, error) { 60 | clientConfig, err := clientcmd.NewClientConfigFromBytes(data) 61 | if err != nil { 62 | return nil, fmt.Errorf("creating client config: %w", err) 63 | } 64 | 65 | return &Getter{ 66 | c: clientConfig, 67 | }, nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/apiloadbalancer/api-loadbalancers_integration_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package apiloadbalancer 5 | 6 | import ( 7 | "os" 8 | "testing" 9 | 10 | "github.com/flexkube/libflexkube/internal/util" 11 | ) 12 | 13 | //nolint:funlen,cyclop // Just lengthy test function. 14 | func TestDeploy(t *testing.T) { 15 | t.Parallel() 16 | 17 | sshPrivateKeyPath := os.Getenv("TEST_INTEGRATION_SSH_PRIVATE_KEY_PATH") 18 | 19 | if sshPrivateKeyPath == "" { 20 | sshPrivateKeyPath = "/home/core/.ssh/id_rsa" 21 | } 22 | 23 | //#nosec G304 // Expected path customization. 24 | key, err := os.ReadFile(sshPrivateKeyPath) 25 | if err != nil { 26 | t.Fatalf("Reading SSH private key shouldn't fail, got: %v", err) 27 | } 28 | 29 | config := ` 30 | servers: 31 | - 10.0.0.2 32 | bindAddress: 0.0.0.0:7443 33 | apiLoadBalancers: 34 | - metricsBindAddress: 0.0.0.0 35 | host: 36 | ssh: 37 | address: localhost 38 | ssh: 39 | user: core 40 | privateKey: |- 41 | ` 42 | 43 | config += util.Indent(string(key), " ") 44 | 45 | if port := os.Getenv("TEST_INTEGRATION_SSH_PORT"); port != "" { 46 | config += " port: " + port 47 | } 48 | 49 | loadBalancers, err := FromYaml([]byte(config)) 50 | if err != nil { 51 | t.Fatalf("Creating apiloadbalancers object should succeed, got: %v", err) 52 | } 53 | 54 | if err := loadBalancers.CheckCurrentState(); err != nil { 55 | t.Fatalf("Checking current state should succeed, got: %v", err) 56 | } 57 | 58 | if err := loadBalancers.Deploy(); err != nil { 59 | t.Fatalf("Deploying should succeed, got: %v", err) 60 | } 61 | 62 | state, err := loadBalancers.StateToYaml() 63 | if err != nil { 64 | t.Fatalf("Dumping state should succeed, got: %v", err) 65 | } 66 | 67 | tearDownConfig := ` 68 | servers: 69 | - 10.0.0.2 70 | apiLoadBalancers: [] 71 | ` 72 | 73 | loadBalancers, err = FromYaml([]byte(tearDownConfig + string(state))) 74 | if err != nil { 75 | t.Fatalf("Creating apiloadbalancers object for teardown should succeed, got: %v", err) 76 | } 77 | 78 | if err := loadBalancers.CheckCurrentState(); err != nil { 79 | t.Fatalf("Checking current state for teardown should succeed, got: %v", err) 80 | } 81 | 82 | if err := loadBalancers.Deploy(); err != nil { 83 | t.Fatalf("Tearing down should succeed, got: %v", err) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pkg/types/privatekey.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "crypto/x509" 5 | "encoding/pem" 6 | "fmt" 7 | "strconv" 8 | 9 | "github.com/flexkube/libflexkube/internal/util" 10 | ) 11 | 12 | // PrivateKey is a wrapper on string type, which parses it's content 13 | // as private key while unmarshaling. This allows to validate the 14 | // data during unmarshaling process. 15 | // 16 | // This type should not be used, as it does not allow to produce 17 | // meaningful error to the user. 18 | type PrivateKey string 19 | 20 | // UnmarshalJSON implements encoding/json.Unmarshaler interface and tries 21 | // to decode obtained data using PEM format and then tries to parse the 22 | // private key as PKCS8, PPKCS1 or EC private keys. 23 | func (p *PrivateKey) UnmarshalJSON(data []byte) error { 24 | unquoted, err := strconv.Unquote(string(data)) 25 | if err != nil { 26 | return fmt.Errorf("unquoting string: %w", err) 27 | } 28 | 29 | der, _ := pem.Decode([]byte(unquoted)) 30 | if der == nil { 31 | return fmt.Errorf("decoding PEM format") 32 | } 33 | 34 | if err := parsePrivateKey(der.Bytes); err != nil { 35 | return fmt.Errorf("parsing private key: %w", err) 36 | } 37 | 38 | *p = PrivateKey(unquoted) 39 | 40 | return nil 41 | } 42 | 43 | // parsePrivateKey tries to parse various private key types and 44 | // returns error if none of them works. 45 | func parsePrivateKey(rawPrivateKey []byte) error { 46 | if _, err := x509.ParsePKCS8PrivateKey(rawPrivateKey); err == nil { 47 | return nil 48 | } 49 | 50 | if _, err := x509.ParsePKCS1PrivateKey(rawPrivateKey); err == nil { 51 | return nil 52 | } 53 | 54 | if _, err := x509.ParseECPrivateKey(rawPrivateKey); err == nil { 55 | return nil 56 | } 57 | 58 | return fmt.Errorf("given key is not a valid PKCS8, PKCS1 or EC private key") 59 | } 60 | 61 | // Pick returns first non-empty private key from given list, including 62 | // receiver private key. 63 | // 64 | // This method is a helper, which allows to select the certificate to use 65 | // from hierarchical configuration. 66 | func (p *PrivateKey) Pick(values ...PrivateKey) PrivateKey { 67 | if p == nil || *p == "" { 68 | pt := PrivateKey("") 69 | p = &pt 70 | } 71 | 72 | pks := []string{string(*p)} 73 | for _, v := range values { 74 | pks = append(pks, string(v)) 75 | } 76 | 77 | return PrivateKey(util.PickString(pks...)) 78 | } 79 | -------------------------------------------------------------------------------- /pkg/types/resource.go: -------------------------------------------------------------------------------- 1 | // Package types provides reusable structs and interfaces used across libflexkube, which 2 | // can also be used by external projects. 3 | package types 4 | 5 | import ( 6 | "fmt" 7 | 8 | "sigs.k8s.io/yaml" 9 | 10 | "github.com/flexkube/libflexkube/pkg/container" 11 | ) 12 | 13 | // Resource interface defines common functionality between Flexkube resources like kubelet pool 14 | // or static controlplane, which allows to manage group of containers. 15 | type Resource interface { 16 | // StateToYaml converts resource's containers state into YAML format and returns it to the user, 17 | // so it can be persisted, e.g. to the file. 18 | StateToYaml() ([]byte, error) 19 | 20 | // CheckCurrentState iterates over containers defined in the state, checks if they exist, are 21 | // running etc and writes to containers current state. This allows then to compare current state 22 | // of the containers with desired state, using Containers() method, to check if there are any 23 | // pending changes to cluster configuration. 24 | // 25 | // Calling CheckCurrentState is required before calling Deploy(), to ensure, that Deploy() executes 26 | // correct actions. 27 | CheckCurrentState() error 28 | 29 | // Deploy creates configured containers. 30 | // 31 | // CheckCurrentState() must be called before calling Deploy(), otherwise error will be returned. 32 | Deploy() error 33 | 34 | // Containers gives access to the ContainersInterface from the resource, which allows accessing 35 | // methods like DesiredState() and ToExported(), which can be used to calculate pending changes 36 | // to the resource configuration. 37 | Containers() container.ContainersInterface 38 | } 39 | 40 | // ResourceConfig interface defines common functionality between all Flexkube resource configurations. 41 | type ResourceConfig interface { 42 | // New creates new Resource object from given configuration and ensures, that the configuration 43 | // is valid. 44 | New() (Resource, error) 45 | 46 | // Validate validates the configuration. 47 | Validate() error 48 | } 49 | 50 | // ResourceFromYaml allows to create any resource instance from YAML configuration. 51 | func ResourceFromYaml(c []byte, r ResourceConfig) (Resource, error) { 52 | if err := yaml.Unmarshal(c, &r); err != nil { 53 | return nil, fmt.Errorf("parsing input YAML: %w", err) 54 | } 55 | 56 | return r.New() 57 | } 58 | -------------------------------------------------------------------------------- /pkg/host/transport/direct/direct_test.go: -------------------------------------------------------------------------------- 1 | package direct_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/pkg/host/transport" 7 | "github.com/flexkube/libflexkube/pkg/host/transport/direct" 8 | ) 9 | 10 | func newDirect(t *testing.T) transport.Interface { 11 | t.Helper() 12 | 13 | d := &direct.Config{} 14 | 15 | di, err := d.New() 16 | if err != nil { 17 | t.Fatalf("Should return new object without errors, got: %v", err) 18 | } 19 | 20 | return di 21 | } 22 | 23 | func TestValidate(t *testing.T) { 24 | t.Parallel() 25 | 26 | d := &direct.Config{} 27 | 28 | if err := d.Validate(); err != nil { 29 | t.Fatalf("Validation should always pass, got: %v", err) 30 | } 31 | } 32 | 33 | func TestForwardUnixSocket(t *testing.T) { 34 | t.Parallel() 35 | 36 | d := newDirect(t) 37 | targetPath := "/foo" 38 | 39 | dc, err := d.Connect() 40 | if err != nil { 41 | t.Fatalf("Connecting: %v", err) 42 | } 43 | 44 | forwardedPath, err := dc.ForwardUnixSocket(targetPath) 45 | if err != nil { 46 | t.Fatalf("Forwarding socket: %v", err) 47 | } 48 | 49 | if forwardedPath != targetPath { 50 | t.Fatalf("Expected %q, got %q", targetPath, forwardedPath) 51 | } 52 | } 53 | 54 | func TestConnect(t *testing.T) { 55 | t.Parallel() 56 | 57 | d := newDirect(t) 58 | 59 | if _, err := d.Connect(); err != nil { 60 | t.Fatalf("Connect should always work, got: %v", err) 61 | } 62 | } 63 | 64 | func TestForwardTCP(t *testing.T) { 65 | t.Parallel() 66 | 67 | d := newDirect(t) 68 | targetAddress := "localhost:80" 69 | 70 | dc, err := d.Connect() 71 | if err != nil { 72 | t.Fatalf("Connecting: %v", err) 73 | } 74 | 75 | forwardedAddress, err := dc.ForwardTCP(targetAddress) 76 | if err != nil { 77 | t.Fatalf("Forwarding TCP: %v", err) 78 | } 79 | 80 | if forwardedAddress != targetAddress { 81 | t.Fatalf("Expected %q, got %q", targetAddress, forwardedAddress) 82 | } 83 | } 84 | 85 | func TestForwardTCPBadAddress(t *testing.T) { 86 | t.Parallel() 87 | 88 | d := newDirect(t) 89 | 90 | directConnected, err := d.Connect() 91 | if err != nil { 92 | t.Fatalf("Connecting: %v", err) 93 | } 94 | 95 | a := "localhost" 96 | 97 | if _, err := directConnected.ForwardTCP(a); err == nil { 98 | t.Fatalf("TCP forwarding should fail when forwarding bad address") 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /pkg/controlplane/validate.go: -------------------------------------------------------------------------------- 1 | package controlplane 2 | 3 | import ( 4 | "fmt" 5 | 6 | "sigs.k8s.io/yaml" 7 | 8 | "github.com/flexkube/libflexkube/internal/util" 9 | "github.com/flexkube/libflexkube/internal/utiltest" 10 | "github.com/flexkube/libflexkube/pkg/host" 11 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 12 | "github.com/flexkube/libflexkube/pkg/types" 13 | ) 14 | 15 | // validator is a helper struct for validating common fields between controlplane 16 | // configuration structs. 17 | type validator struct { 18 | Common *Common `json:"common,omitempty"` 19 | Host *host.Host `json:"host,omitempty"` 20 | Kubeconfig client.Config `json:"kubeconfig,omitempty"` 21 | YAML interface{} `json:"yaml,omitempty"` 22 | } 23 | 24 | // validate validates validator struct. If validateKubeconfig is false, 25 | // then validation of this field will be skipped. 26 | func (v validator) validate(validateKubeconfig bool) error { 27 | var errors util.ValidateErrors 28 | 29 | if !validateKubeconfig { 30 | pki, err := utiltest.GeneratePKIErr() 31 | if err != nil { 32 | return fmt.Errorf("generating fake PKI for validation: %w", err) 33 | } 34 | 35 | v.Kubeconfig = client.Config{ 36 | Server: "localhost", 37 | CACertificate: types.Certificate(pki.Certificate), 38 | ClientCertificate: types.Certificate(pki.Certificate), 39 | ClientKey: types.PrivateKey(pki.PrivateKey), 40 | } 41 | } 42 | 43 | b, err := yaml.Marshal(v) 44 | if err != nil { 45 | return append(errors, fmt.Errorf("marshaling kubeconfig: %w", err)) 46 | } 47 | 48 | if err := yaml.Unmarshal(b, &v); err != nil { 49 | return append(errors, fmt.Errorf("unmarshaling kubeconfig: %w", err)) 50 | } 51 | 52 | if v.Common == nil { 53 | errors = append(errors, fmt.Errorf("common certificates must not defined")) 54 | } 55 | 56 | if validateKubeconfig { 57 | if _, err := v.Kubeconfig.ToYAMLString(); err != nil { 58 | errors = append(errors, fmt.Errorf("invalid kubeconfig: %w", err)) 59 | } 60 | } 61 | 62 | errors = append(errors, v.validateHost()...) 63 | 64 | return errors.Return() 65 | } 66 | 67 | func (v validator) validateHost() util.ValidateErrors { 68 | var errors util.ValidateErrors 69 | 70 | if v.Host == nil { 71 | errors = append(errors, fmt.Errorf("host must be defined")) 72 | } 73 | 74 | if v.Host != nil { 75 | if err := v.Host.Validate(); err != nil { 76 | errors = append(errors, fmt.Errorf("validating host configuration: %w", err)) 77 | } 78 | } 79 | 80 | return errors 81 | } 82 | -------------------------------------------------------------------------------- /pkg/apiloadbalancer/api-loadbalancers_test.go: -------------------------------------------------------------------------------- 1 | package apiloadbalancer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/pkg/types" 7 | ) 8 | 9 | func TestPoolNoInstancesDefined(t *testing.T) { 10 | t.Parallel() 11 | 12 | a := &APILoadBalancers{} 13 | 14 | if err := a.Validate(); err == nil { 15 | t.Fatal("Validate should fail if there is no instances defined and the state is empty") 16 | } 17 | } 18 | 19 | func GetLoadBalancers(t *testing.T) types.Resource { 20 | t.Helper() 21 | 22 | testConfigRaw := ` 23 | ssh: 24 | address: localhost 25 | password: foo 26 | connectionTimeout: 1s 27 | retryTimeout: 1s 28 | retryInterval: 1s 29 | bindAddress: 0.0.0.0:6443 30 | apiLoadBalancers: 31 | - {} 32 | servers: 33 | - localhost:6443 34 | ` 35 | 36 | p, err := FromYaml([]byte(testConfigRaw)) 37 | if err != nil { 38 | t.Fatalf("Creating load balancers from YAML should succeed, got: %v", err) 39 | } 40 | 41 | return p 42 | } 43 | 44 | // New() tests. 45 | func TestLoadBalancersNewValidate(t *testing.T) { 46 | t.Parallel() 47 | 48 | testConfigRaw := ` 49 | ssh: 50 | address: localhost 51 | password: foo 52 | connectionTimeout: 1s 53 | retryTimeout: 1s 54 | retryInterval: 1s 55 | bindAddress: 0.0.0.0:6443 56 | apiLoadBalancers: 57 | - {} 58 | ` 59 | 60 | if _, err := FromYaml([]byte(testConfigRaw)); err == nil { 61 | t.Fatalf("Creating load balancers from bad YAML should fail") 62 | } 63 | } 64 | 65 | // FromYaml() tests. 66 | func TestLoadBalancersFromYaml(t *testing.T) { 67 | t.Parallel() 68 | GetLoadBalancers(t) 69 | } 70 | 71 | // StateToYaml() tests. 72 | func TestLoadBalancersStateToYAML(t *testing.T) { 73 | t.Parallel() 74 | 75 | p := GetLoadBalancers(t) 76 | 77 | if _, err := p.StateToYaml(); err != nil { 78 | t.Fatalf("Dumping state to YAML should work, got: %v", err) 79 | } 80 | } 81 | 82 | // CheckCurrentState() tests. 83 | func TestLoadBalancersCheckCurrentState(t *testing.T) { 84 | t.Parallel() 85 | 86 | p := GetLoadBalancers(t) 87 | 88 | if err := p.CheckCurrentState(); err != nil { 89 | t.Fatalf("Dumping state to YAML should work, got: %v", err) 90 | } 91 | } 92 | 93 | // Deploy() tests. 94 | func TestLoadBalancersDeploy(t *testing.T) { 95 | t.Parallel() 96 | 97 | p := GetLoadBalancers(t) 98 | 99 | if err := p.Deploy(); err == nil { 100 | t.Fatalf("Deploying in testing environment should fail") 101 | } 102 | } 103 | 104 | // Containers() tests. 105 | func TestLoadBalancersContainers(t *testing.T) { 106 | t.Parallel() 107 | 108 | p := GetLoadBalancers(t) 109 | 110 | if c := p.Containers(); c == nil { 111 | t.Fatalf("Containers() should return non-nil value") 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /pkg/kubernetes/client/client_test.go: -------------------------------------------------------------------------------- 1 | package client_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | 8 | "k8s.io/apimachinery/pkg/util/wait" 9 | 10 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 11 | ) 12 | 13 | func TestCheckNodeExistsFakeKubeconfig(t *testing.T) { 14 | t.Parallel() 15 | 16 | kubeconfig := GetKubeconfig(t) 17 | 18 | c, err := client.NewClient([]byte(kubeconfig)) 19 | if err != nil { 20 | t.Fatalf("Failed creating client: %v", err) 21 | } 22 | 23 | e, err := c.CheckNodeExists("foo")() 24 | 25 | if e == true { 26 | t.Errorf("Node should never exists with fake kubeconfig") 27 | } 28 | 29 | if err == nil { 30 | t.Errorf("Checking node existence should always fail with fake kubeconfig") 31 | } 32 | } 33 | 34 | func TestWaitForNodeFakeKubeconfig(t *testing.T) { 35 | t.Parallel() 36 | 37 | kubeconfig := GetKubeconfig(t) 38 | 39 | c, err := client.NewClient([]byte(kubeconfig)) 40 | if err != nil { 41 | t.Fatalf("Failed creating client: %v", err) 42 | } 43 | 44 | if err := c.WaitForNode("foo"); err == nil { 45 | t.Errorf("Waiting for node should always fail with fake kubeconfig") 46 | } 47 | } 48 | 49 | func TestLabelNodeFakeKubeconfig(t *testing.T) { 50 | t.Parallel() 51 | 52 | kubeconfig := GetKubeconfig(t) 53 | 54 | testClient, err := client.NewClient([]byte(kubeconfig)) 55 | if err != nil { 56 | t.Fatalf("Failed creating client: %v", err) 57 | } 58 | 59 | l := map[string]string{ 60 | "foo": "bar", 61 | } 62 | 63 | if err := testClient.LabelNode("foo", l); err == nil { 64 | t.Errorf("Labeling node should always fail with fake kubeconfig") 65 | } 66 | } 67 | 68 | // PingWait() tests. 69 | func TestPingWaitFakeKubeconfig(t *testing.T) { 70 | t.Parallel() 71 | 72 | kubeconfig := GetKubeconfig(t) 73 | 74 | testClient, err := client.NewClient([]byte(kubeconfig)) 75 | if err != nil { 76 | t.Fatalf("Failed creating client: %v", err) 77 | } 78 | 79 | //nolint:staticcheck // Will migrate once https://github.com/kubernetes/kubernetes/issues/119533 is resolved. 80 | if err := testClient.PingWait(1*time.Second, 1*time.Second); !errors.Is(err, wait.ErrWaitTimeout) { 81 | t.Fatalf("Ping with fake config should always timeout, got: %v", err) 82 | } 83 | } 84 | 85 | // CheckNodeReady() tests. 86 | func TestCheckNodeReadyFakeKubeconfig(t *testing.T) { 87 | t.Parallel() 88 | 89 | kubeconfig := GetKubeconfig(t) 90 | 91 | c, err := client.NewClient([]byte(kubeconfig)) 92 | if err != nil { 93 | t.Fatalf("Failed creating client: %v", err) 94 | } 95 | 96 | e, err := c.CheckNodeReady("foo")() 97 | 98 | if e == true { 99 | t.Errorf("Check should never return true with fake kubeconfig") 100 | } 101 | 102 | if err != nil { 103 | t.Errorf("Check should swallow all errors and just return boolean value") 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /pkg/container/runtime/fake.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flexkube/libflexkube/pkg/container/types" 8 | ) 9 | 10 | // Fake is a fake runtime client, which can be used for testing. 11 | type Fake struct { 12 | // CreateF will be Create by method. 13 | CreateF func(config *types.ContainerConfig) (string, error) 14 | 15 | // DeleteF will be called by Delete method. 16 | DeleteF func(id string) error 17 | 18 | // StartF will be called by Start method. 19 | StartF func(id string) error 20 | 21 | // StatusF will be called by Status method. 22 | StatusF func(id string) (types.ContainerStatus, error) 23 | 24 | // StopF will be called by Stop method. 25 | StopF func(id string) error 26 | 27 | // CopyF will be called by Copy method. 28 | CopyF func(id string, files []*types.File) error 29 | 30 | // ReadF will be called by Read method. 31 | ReadF func(id string, srcPath []string) ([]*types.File, error) 32 | 33 | // StatF will be called by Stat method. 34 | StatF func(id string, paths []string) (map[string]os.FileMode, error) 35 | } 36 | 37 | // Create mocks runtime Create(). 38 | func (f Fake) Create(config *types.ContainerConfig) (string, error) { 39 | return f.CreateF(config) 40 | } 41 | 42 | // Delete mocks runtime Delete(). 43 | func (f Fake) Delete(id string) error { 44 | return f.DeleteF(id) 45 | } 46 | 47 | // Start mocks runtime Start(). 48 | func (f Fake) Start(id string) error { 49 | return f.StartF(id) 50 | } 51 | 52 | // Status mocks runtime Status(). 53 | func (f Fake) Status(id string) (types.ContainerStatus, error) { 54 | return f.StatusF(id) 55 | } 56 | 57 | // Stop mocks runtime Stop(). 58 | func (f Fake) Stop(id string) error { 59 | return f.StopF(id) 60 | } 61 | 62 | // Copy mocks runtime Copy(). 63 | func (f Fake) Copy(id string, files []*types.File) error { 64 | return f.CopyF(id, files) 65 | } 66 | 67 | // Read mocks runtime Read(). 68 | func (f Fake) Read(id string, srcPath []string) ([]*types.File, error) { 69 | return f.ReadF(id, srcPath) 70 | } 71 | 72 | // Stat mocks runtime Stat(). 73 | func (f Fake) Stat(id string, paths []string) (map[string]os.FileMode, error) { 74 | return f.StatF(id, paths) 75 | } 76 | 77 | // FakeConfig is a Fake runtime configuration struct. 78 | type FakeConfig struct { 79 | // Runtime holds container runtime to return by New() method. 80 | Runtime Runtime 81 | 82 | // Address will be used for GetAddress and SetAddress methods. 83 | Address string 84 | } 85 | 86 | // GetAddress implements runtime.Config interface. 87 | func (c FakeConfig) GetAddress() string { 88 | return c.Address 89 | } 90 | 91 | // SetAddress implements runtime.Config interface. 92 | func (c *FakeConfig) SetAddress(a string) { 93 | c.Address = a 94 | } 95 | 96 | // New implements runtime.Config interface. 97 | func (c FakeConfig) New() (Runtime, error) { 98 | if c.Runtime == nil { 99 | return nil, fmt.Errorf("no runtime defined") 100 | } 101 | 102 | return c.Runtime, nil 103 | } 104 | -------------------------------------------------------------------------------- /pkg/types/privatekey_test.go: -------------------------------------------------------------------------------- 1 | package types_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "sigs.k8s.io/yaml" 9 | 10 | "github.com/flexkube/libflexkube/internal/util" 11 | "github.com/flexkube/libflexkube/internal/utiltest" 12 | "github.com/flexkube/libflexkube/pkg/types" 13 | ) 14 | 15 | type Foo struct { 16 | Bar types.PrivateKey `json:"bar"` 17 | } 18 | 19 | func TestPrivateKeyParse(t *testing.T) { 20 | t.Parallel() 21 | 22 | cases := map[string]struct { 23 | YAML string 24 | Error bool 25 | }{ 26 | "bad": { 27 | "bar: doh", 28 | true, 29 | }, 30 | "good": { 31 | fmt.Sprintf("bar: |\n%s", util.Indent(strings.TrimSpace(utiltest.GenerateRSAPrivateKey(t)), " ")), 32 | false, 33 | }, 34 | } 35 | 36 | for n, testCase := range cases { 37 | testCase := testCase 38 | 39 | t.Run(n, func(t *testing.T) { 40 | t.Parallel() 41 | 42 | bar := &Foo{} 43 | 44 | err := yaml.Unmarshal([]byte(testCase.YAML), bar) 45 | if testCase.Error && err == nil { 46 | t.Fatalf("Expected error and didn't get any.") 47 | } 48 | 49 | if !testCase.Error && err != nil { 50 | t.Fatalf("Didn't expect error, got: %v", err) 51 | } 52 | 53 | if err == nil && bar.Bar == "" { 54 | t.Fatalf("Didn't get any error, but field is empty") 55 | } 56 | 57 | if err != nil && bar.Bar != "" { 58 | t.Fatalf("Got error and still got some content") 59 | } 60 | }) 61 | } 62 | } 63 | 64 | func TestParsePrivateKeyPKCS1(t *testing.T) { 65 | t.Parallel() 66 | 67 | d := fmt.Sprintf("bar: |\n%s", util.Indent(strings.TrimSpace(utiltest.GeneratePKCS1PrivateKey(t)), " ")) 68 | 69 | if err := yaml.Unmarshal([]byte(d), &Foo{}); err != nil { 70 | t.Fatalf("Parsing valid PKCS1 private key should succeed, got: %v", err) 71 | } 72 | } 73 | 74 | func TestParsePrivateKeyEC(t *testing.T) { 75 | t.Parallel() 76 | 77 | d := fmt.Sprintf("bar: |\n%s", util.Indent(strings.TrimSpace(utiltest.GenerateECPrivateKey(t)), " ")) 78 | 79 | if err := yaml.Unmarshal([]byte(d), &Foo{}); err != nil { 80 | t.Fatalf("Parsing valid EC private key should succeed, got: %v", err) 81 | } 82 | } 83 | 84 | func TestParsePrivateKeyBad(t *testing.T) { 85 | t.Parallel() 86 | 87 | //#nosec G101 // Just bad test data. 88 | privateKey := `--- 89 | bar: | 90 | -----BEGIN RSA PRIVATE KEY----- 91 | Zm9vCg== 92 | -----END RSA PRIVATE KEY----- 93 | ` 94 | if err := yaml.Unmarshal([]byte(privateKey), &Foo{}); err == nil { 95 | t.Fatalf("Parsing not PEM format should fail") 96 | } 97 | } 98 | 99 | func TestPrivateKeyPickNil(t *testing.T) { 100 | t.Parallel() 101 | 102 | var c types.PrivateKey 103 | 104 | if c.Pick(types.PrivateKey("bar"), types.PrivateKey("baz")) != "bar" { 105 | t.Fatalf("First non empty private key should be picked") 106 | } 107 | } 108 | 109 | func TestPrivateKeyPick(t *testing.T) { 110 | t.Parallel() 111 | 112 | d := types.PrivateKey("foo") 113 | 114 | if d.Pick(types.PrivateKey("baz")) != "foo" { 115 | t.Fatalf("First non empty private key should be picked") 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /internal/util/util.go: -------------------------------------------------------------------------------- 1 | // Package util provides very generic helpers used across codebase. 2 | package util 3 | 4 | import ( 5 | "fmt" 6 | "sort" 7 | "strings" 8 | 9 | "github.com/logrusorgru/aurora" 10 | ) 11 | 12 | // PickString returns first non-empty string passed. 13 | func PickString(values ...string) string { 14 | for _, v := range values { 15 | if v != "" { 16 | return v 17 | } 18 | } 19 | 20 | return "" 21 | } 22 | 23 | // PickStringSlice returns first non-empty strings slice. 24 | func PickStringSlice(values ...[]string) []string { 25 | for _, v := range values { 26 | if len(v) > 0 { 27 | return v 28 | } 29 | } 30 | 31 | return []string{} 32 | } 33 | 34 | // PickStringMap returns first non-empty map of strings. 35 | func PickStringMap(values ...map[string]string) map[string]string { 36 | for _, v := range values { 37 | if len(v) > 0 { 38 | return v 39 | } 40 | } 41 | 42 | return map[string]string{} 43 | } 44 | 45 | // PickInt returns first non-zero integer passed. 46 | func PickInt(values ...int) int { 47 | for _, v := range values { 48 | if v != 0 { 49 | return v 50 | } 51 | } 52 | 53 | return 0 54 | } 55 | 56 | // Indent indents a block of text with an indent string. 57 | func Indent(text, indent string) string { 58 | if text == "" { 59 | return "" 60 | } 61 | 62 | if text[len(text)-1:] == "\n" { 63 | result := "" 64 | for _, j := range strings.Split(text[:len(text)-1], "\n") { 65 | result += indent + j + "\n" 66 | } 67 | 68 | return result 69 | } 70 | 71 | result := "" 72 | 73 | for _, j := range strings.Split(strings.TrimRight(text, "\n"), "\n") { 74 | result += indent + j + "\n" 75 | } 76 | 77 | return result[:len(result)-1] 78 | } 79 | 80 | // JoinSorted takes map of keys and values, sorts them by keys and joins with given separators. 81 | func JoinSorted(values map[string]string, valueSeparator, keySeparator string) string { 82 | keys := []string{} 83 | 84 | for k := range values { 85 | keys = append(keys, k) 86 | } 87 | 88 | sort.Strings(keys) 89 | 90 | formattedKeys := []string{} 91 | 92 | for _, k := range keys { 93 | formattedKeys = append(formattedKeys, fmt.Sprintf("%s%s%s", k, valueSeparator, values[k])) 94 | } 95 | 96 | return strings.Join(formattedKeys, keySeparator) 97 | } 98 | 99 | // KeysStringMap returns keys from given map. 100 | func KeysStringMap(m map[string]string) []string { 101 | keys := []string{} 102 | 103 | for k := range m { 104 | keys = append(keys, k) 105 | } 106 | 107 | sort.Strings(keys) 108 | 109 | return keys 110 | } 111 | 112 | // ColorizeDiff takes diff-formatter output and adds console colors to it. 113 | func ColorizeDiff(diff string) string { 114 | // Don't even try to process empty strings. 115 | if diff == "" { 116 | return diff 117 | } 118 | 119 | // If string ends with newline, strip it before splitting, then we add it at the end of processing. 120 | endsWithNewLine := diff[len(diff)-1] == '\n' 121 | if endsWithNewLine { 122 | diff = diff[:len(diff)-1] 123 | } 124 | 125 | lines := strings.Split(diff, "\n") 126 | linesCount := len(lines) 127 | 128 | output := "" 129 | 130 | for i, line := range strings.Split(diff, "\n") { 131 | coloredLine := line + "\n" 132 | 133 | // If we process last line and the given diff does not end with newline, don't include it. 134 | if !endsWithNewLine && i == linesCount-1 { 135 | coloredLine = line 136 | } 137 | 138 | if len(line) > 0 && line[0] == '-' { 139 | coloredLine = aurora.Red(line + "\n").String() 140 | } 141 | 142 | if len(line) > 0 && line[0] == '+' { 143 | coloredLine = aurora.Green(line + "\n").String() 144 | } 145 | 146 | output += coloredLine 147 | } 148 | 149 | return output 150 | } 151 | -------------------------------------------------------------------------------- /libvirt/main.tf: -------------------------------------------------------------------------------- 1 | provider "libvirt" { 2 | uri = "qemu:///system" 3 | } 4 | 5 | variable "core_public_keys" { 6 | type = list(string) 7 | } 8 | 9 | resource "libvirt_pool" "pool" { 10 | name = "libflexkube" 11 | type = "dir" 12 | path = "${abspath(path.module)}/pool" 13 | } 14 | 15 | resource "libvirt_volume" "base" { 16 | name = "flexkube-base" 17 | source = "${abspath(path.module)}/flatcar_production_qemu_image.img" 18 | pool = libvirt_pool.pool.name 19 | format = "qcow2" 20 | } 21 | 22 | resource "libvirt_network" "network" { 23 | name = "flexkube" 24 | mode = "nat" 25 | domain = "k8s.local" 26 | addresses = [var.nodes_cidr] 27 | 28 | dns { 29 | local_only = true 30 | # can specify local names here 31 | } 32 | } 33 | 34 | data "ct_config" "controller" { 35 | count = var.controllers_count 36 | 37 | content = templatefile("./templates/ct_config.yaml.tmpl", { 38 | core_public_keys = var.core_public_keys 39 | hostname = local.controller_names[count.index] 40 | }) 41 | } 42 | 43 | resource "libvirt_ignition" "controller" { 44 | count = var.controllers_count 45 | 46 | name = "flexkube-ignition-controller-${count.index}" 47 | pool = libvirt_pool.pool.name 48 | 49 | content = data.ct_config.controller[count.index].rendered 50 | } 51 | 52 | resource "libvirt_volume" "controller-disk" { 53 | count = var.controllers_count 54 | 55 | name = local.controller_names[count.index] 56 | base_volume_id = libvirt_volume.base.id 57 | pool = libvirt_pool.pool.name 58 | format = "qcow2" 59 | } 60 | 61 | resource "libvirt_domain" "controller_machine" { 62 | count = var.controllers_count 63 | name = local.controller_names[count.index] 64 | vcpu = 2 65 | memory = 4096 66 | 67 | disk { 68 | volume_id = libvirt_volume.controller-disk[count.index].id 69 | } 70 | 71 | fw_cfg_name = "opt/org.flatcar-linux/config" 72 | coreos_ignition = libvirt_ignition.controller[count.index].id 73 | 74 | graphics { 75 | listen_type = "address" 76 | } 77 | 78 | network_interface { 79 | network_id = libvirt_network.network.id 80 | hostname = local.controller_names[count.index] 81 | addresses = [local.controller_ips[count.index]] 82 | } 83 | } 84 | 85 | resource "libvirt_volume" "worker-disk" { 86 | count = var.workers_count 87 | 88 | name = local.worker_names[count.index] 89 | base_volume_id = libvirt_volume.base.id 90 | pool = libvirt_pool.pool.name 91 | format = "qcow2" 92 | } 93 | 94 | data "ct_config" "worker" { 95 | count = var.workers_count 96 | 97 | content = templatefile("./templates/ct_config.yaml.tmpl", { 98 | core_public_keys = var.core_public_keys 99 | hostname = local.worker_names[count.index] 100 | }) 101 | } 102 | 103 | resource "libvirt_ignition" "worker" { 104 | count = var.workers_count 105 | 106 | name = "flexkube-ignition-worker-${count.index}" 107 | pool = libvirt_pool.pool.name 108 | 109 | content = data.ct_config.worker[count.index].rendered 110 | } 111 | 112 | resource "libvirt_domain" "worker_machine" { 113 | count = var.workers_count 114 | name = local.worker_names[count.index] 115 | vcpu = 2 116 | memory = 2048 117 | 118 | disk { 119 | volume_id = libvirt_volume.worker-disk[count.index].id 120 | } 121 | 122 | fw_cfg_name = "opt/org.flatcar-linux/config" 123 | coreos_ignition = libvirt_ignition.worker[count.index].id 124 | 125 | graphics { 126 | listen_type = "address" 127 | } 128 | 129 | network_interface { 130 | network_id = libvirt_network.network.id 131 | hostname = local.worker_names[count.index] 132 | addresses = [local.worker_ips[count.index]] 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /pkg/container/resource/resource.go: -------------------------------------------------------------------------------- 1 | // Package resource stores a wrapper over container.Containers, which implements 2 | // types.Resource interface. It is stored in separate package to avoid cyclic imports. 3 | package resource 4 | 5 | import ( 6 | "fmt" 7 | 8 | "sigs.k8s.io/yaml" 9 | 10 | "github.com/flexkube/libflexkube/pkg/container" 11 | "github.com/flexkube/libflexkube/pkg/types" 12 | ) 13 | 14 | // Containers is a wrapper over container.Containers, which implemens types.ResourceConfig 15 | // and also has JSON field tags the same as other resources. 16 | // 17 | // It allows to orchestrate and update multiple containers spread 18 | // across multiple hosts and update their configurations. 19 | type Containers struct { 20 | // State holds containers state. 21 | State container.ContainersState `json:"state,omitempty"` 22 | 23 | // Containers stores user-provider containers to create. 24 | Containers container.ContainersState `json:"containers,omitempty"` 25 | } 26 | 27 | // containers implements both container.ContainersInterface and types.Resource. 28 | type containers struct { 29 | containers container.ContainersInterface 30 | } 31 | 32 | // New creates new containers instance, but returns generic types.Resource type. 33 | // 34 | // This method will validate all the configuration provided. 35 | func (c *Containers) New() (types.Resource, error) { 36 | containersConfig := container.Containers{ 37 | PreviousState: c.State, 38 | DesiredState: c.Containers, 39 | } 40 | 41 | newContainers, err := containersConfig.New() 42 | if err != nil { 43 | return nil, fmt.Errorf("creating containers object: %w", err) 44 | } 45 | 46 | return &containers{ 47 | containers: newContainers, 48 | }, nil 49 | } 50 | 51 | // Validate validates Containers configuration. 52 | // 53 | // Validate is also part of types.ResourceConfig interface. 54 | func (c *Containers) Validate() error { 55 | co := container.Containers{ 56 | PreviousState: c.State, 57 | DesiredState: c.Containers, 58 | } 59 | 60 | return co.Validate() 61 | } 62 | 63 | // StateToYaml serializes containers state to YAML format. 64 | func (c *containers) StateToYaml() ([]byte, error) { 65 | co := &Containers{ 66 | State: c.containers.ToExported().PreviousState, 67 | } 68 | 69 | return yaml.Marshal(co) 70 | } 71 | 72 | // CheckCurrentState is part of container.ContainersInterface. 73 | func (c *containers) CheckCurrentState() error { 74 | return c.containers.CheckCurrentState() 75 | } 76 | 77 | // Deploy creates configured containers. 78 | // 79 | // CheckCurrentState() must be called before calling Deploy(), otherwise error will be returned. 80 | // 81 | // Deploy is part of container.ContainersInterface. 82 | func (c *containers) Deploy() error { 83 | return c.containers.Deploy() 84 | } 85 | 86 | // ToExported converts unexported containers struct into exported one, which can be then 87 | // serialized and persisted. 88 | // 89 | // ToExported is part of container.ContainersInterface. 90 | func (c *containers) ToExported() *container.Containers { 91 | return c.containers.ToExported() 92 | } 93 | 94 | // DesiredState returns desired state of configured containers. 95 | // 96 | // Desired state differs from 97 | // exported or user-defined desired state, as it will have container IDs filled from the 98 | // previous state. 99 | // 100 | // All returned containers will also have status set to running, as this is always the desired 101 | // state of the container. 102 | // 103 | // Having those fields modified allows to minimize the difference when comparing previous state 104 | // and desired state. 105 | // 106 | // DesiredState is part of container.ContainersInterface. 107 | func (c *containers) DesiredState() container.ContainersState { 108 | return c.containers.DesiredState() 109 | } 110 | 111 | // Containers is part of container.ContainersInterface. 112 | func (c *containers) Containers() container.ContainersInterface { 113 | return c.containers 114 | } 115 | -------------------------------------------------------------------------------- /pkg/controlplane/kube-scheduler_test.go: -------------------------------------------------------------------------------- 1 | package controlplane 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/internal/utiltest" 7 | "github.com/flexkube/libflexkube/pkg/host" 8 | "github.com/flexkube/libflexkube/pkg/host/transport/direct" 9 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 10 | "github.com/flexkube/libflexkube/pkg/types" 11 | ) 12 | 13 | func TestKubeSchedulerToHostConfiguredContainer(t *testing.T) { 14 | t.Parallel() 15 | 16 | pki := utiltest.GeneratePKI(t) 17 | 18 | kubeScheduler := &KubeScheduler{ 19 | Common: &Common{ 20 | FrontProxyCACertificate: types.Certificate(pki.Certificate), 21 | }, 22 | Kubeconfig: client.Config{ 23 | Server: "localhost", 24 | CACertificate: types.Certificate(pki.Certificate), 25 | ClientCertificate: types.Certificate(pki.Certificate), 26 | ClientKey: types.PrivateKey(pki.PrivateKey), 27 | }, 28 | Host: &host.Host{ 29 | DirectConfig: &direct.Config{}, 30 | }, 31 | } 32 | 33 | o, err := kubeScheduler.New() 34 | if err != nil { 35 | t.Fatalf("New should not return error, got: %v", err) 36 | } 37 | 38 | hcc, err := o.ToHostConfiguredContainer() 39 | if err != nil { 40 | t.Fatalf("Generating HostConfiguredContainer should work, got: %v", err) 41 | } 42 | 43 | if _, err := hcc.New(); err != nil { 44 | t.Fatalf("ToHostConfiguredContainer() should generate valid HostConfiguredContainer, got: %v", err) 45 | } 46 | 47 | if hcc.Container.Config.Image == "" { 48 | t.Fatalf("New() should set default image if it's not present") 49 | } 50 | } 51 | 52 | // New() tests. 53 | func TestKubeSchedulerNewEmptyHost(t *testing.T) { 54 | t.Parallel() 55 | 56 | ks := &KubeScheduler{} 57 | 58 | k, err := ks.New() 59 | if err == nil { 60 | t.Errorf("Attempting to create kube-scheduler from empty config should fail") 61 | } 62 | 63 | if k != nil { 64 | t.Fatalf("Failed attempt of creating kube-scheduler should not return kube-scheduler object") 65 | } 66 | } 67 | 68 | // Validate() tests. 69 | // 70 | //nolint:funlen // Just many test cases. 71 | func TestKubeSchedulerValidate(t *testing.T) { 72 | t.Parallel() 73 | 74 | pki := utiltest.GeneratePKI(t) 75 | 76 | hostConfig := &host.Host{ 77 | DirectConfig: &direct.Config{}, 78 | } 79 | 80 | common := &Common{ 81 | KubernetesCACertificate: types.Certificate(pki.Certificate), 82 | FrontProxyCACertificate: types.Certificate(pki.Certificate), 83 | } 84 | 85 | kubeconfig := client.Config{ 86 | Server: "localhost", 87 | CACertificate: types.Certificate(pki.Certificate), 88 | ClientCertificate: types.Certificate(pki.Certificate), 89 | ClientKey: types.PrivateKey(pki.PrivateKey), 90 | } 91 | 92 | cases := map[string]struct { 93 | Config *KubeScheduler 94 | Error bool 95 | }{ 96 | "require common certificates": { 97 | Config: &KubeScheduler{ 98 | Host: hostConfig, 99 | Kubeconfig: kubeconfig, 100 | }, 101 | Error: true, 102 | }, 103 | "validate kubeletClientCertificate": { 104 | Config: &KubeScheduler{ 105 | Common: common, 106 | Host: hostConfig, 107 | }, 108 | Error: true, 109 | }, 110 | "validate host": { 111 | Config: &KubeScheduler{ 112 | Common: common, 113 | Kubeconfig: kubeconfig, 114 | Host: &host.Host{}, 115 | }, 116 | Error: true, 117 | }, 118 | "valid": { 119 | Config: &KubeScheduler{ 120 | Common: common, 121 | Kubeconfig: kubeconfig, 122 | Host: hostConfig, 123 | }, 124 | Error: false, 125 | }, 126 | } 127 | 128 | for n, testCase := range cases { 129 | testCase := testCase 130 | 131 | t.Run(n, func(t *testing.T) { 132 | t.Parallel() 133 | 134 | err := testCase.Config.Validate() 135 | if !testCase.Error && err != nil { 136 | t.Errorf("Didn't expect error, got: %v", err) 137 | } 138 | 139 | if testCase.Error && err == nil { 140 | t.Errorf("Expected error") 141 | } 142 | }) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | Vagrant.configure("2") do |config| 4 | channel = ENV["TF_VAR_flatcar_channel"] || "edge" 5 | 6 | # Box setup. 7 | config.vm.box = "flatcar-#{channel}" 8 | config.vm.box_url = "https://#{channel}.release.flatcar-linux.net/amd64-usr/current/flatcar_production_vagrant.box" 9 | 10 | # Virtualbox resources. 11 | config.vm.provider :virtualbox do |v| 12 | v.check_guest_additions = false 13 | v.functional_vboxsf = false 14 | v.cpus = 2 15 | v.memory = 2048 16 | v.customize ['modifyvm', :id, '--paravirtprovider', 'kvm'] 17 | end 18 | 19 | # SSH. 20 | config.ssh.username = 'core' 21 | 22 | # Number of controllers and workers. 23 | controllers = ENV["TF_VAR_controllers_count"].to_i || 1 24 | workers = ENV["TF_VAR_workers_count"].to_i || 0 25 | nodes_cidr = (ENV["TF_VAR_nodes_cidr"] || "192.168.50.0/24").split("/")[0].split(".")[0..2].join(".") + "." 26 | 27 | # Make sure there is only one primary VM. 28 | primary = true 29 | 30 | # Also don't use Virtualbox DNS server, as it does not support TCP queries it seems and this makes CoreDNS complains. 31 | network_config = <<-EOF 32 | [Match] 33 | Name=eth0 34 | 35 | [Network] 36 | DHCP=yes 37 | DNS=8.8.8.8 38 | DNS=8.8.4.4 39 | 40 | [DHCP] 41 | UseDNS=false 42 | EOF 43 | 44 | common_provisioning_script = <<-EOF 45 | mkdir -p /etc/systemd/network && echo "#{network_config}" | sudo tee /etc/systemd/network/10-virtualbox.network >/dev/null 46 | sudo systemctl daemon-reload 47 | sudo systemctl enable iptables-store iptables-restore docker containerd systemd-timesyncd 48 | sudo systemctl stop update-engine locksmithd 49 | sudo systemctl mask update-engine locksmithd 50 | sudo systemctl start docker systemd-timesyncd iptables-store 51 | sudo systemctl restart systemd-networkd 52 | EOF 53 | 54 | # Controllers. 55 | (1..controllers).each do |i| 56 | config.vm.define vm_name = "controller%02d" % i, primary: primary do |config| 57 | # Set hostname 58 | config.vm.hostname = vm_name 59 | 60 | config.vm.network "private_network", ip: nodes_cidr + (i+1).to_s 61 | 62 | if primary 63 | primary = false 64 | 65 | config.vm.provider :virtualbox do |v| 66 | v.cpus = 6 67 | v.memory = 4096 68 | end 69 | 70 | # Sync using rsync, but don't copy locally built binaries and don't remove Terraform files from virtual machine. 71 | config.vm.synced_folder ".", "/home/core/libflexkube", type: "rsync", rsync__exclude: [ 72 | ".git/", 73 | "bin/", 74 | "e2e/config.yaml", 75 | "e2e/test-config.yaml", 76 | "e2e/state.yaml", 77 | "e2e/resources", 78 | "e2e/values", 79 | "e2e/kubeconfig", 80 | "libvirt", 81 | ] 82 | 83 | # Read content of Vagrant SSH private key. 84 | ssh_private_key = File.read(ENV['HOME'] + "/.vagrant.d/insecure_private_key") 85 | 86 | # Primary node provisioning. 87 | config.vm.provision "shell", inline: <<-EOF 88 | set -e 89 | echo "#{ssh_private_key}" > /home/core/.ssh/id_rsa && chmod 0600 /home/core/.ssh/id_rsa 90 | openssl rand -base64 14 > /home/core/.ssh/password 91 | yes $(cat /home/core/.ssh/password) | sudo passwd core 92 | EOF 93 | end 94 | 95 | # Controller provisioning. 96 | config.vm.provision "shell", inline: <<-EOF 97 | set -e 98 | #{common_provisioning_script} 99 | EOF 100 | end 101 | end 102 | 103 | # Workers. 104 | (1..workers).each do |i| 105 | config.vm.define vm_name = "worker%02d" % i do |config| 106 | config.vm.hostname = vm_name 107 | config.vm.network "private_network", ip: nodes_cidr + (i+1+10).to_s 108 | 109 | # Provisioning. 110 | config.vm.provision "shell", inline: <<-EOF 111 | set -e 112 | #{common_provisioning_script} 113 | EOF 114 | end 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /pkg/helm/release/release_test.go: -------------------------------------------------------------------------------- 1 | package release_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/pkg/helm/release" 7 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 8 | "github.com/flexkube/libflexkube/pkg/pki" 9 | ) 10 | 11 | // New() tests. 12 | // 13 | //nolint:paralleltest // Helm client is not thread-safe. 14 | func TestConfigNewBadKubeconfig(t *testing.T) { 15 | config := &release.Config{ 16 | // Put content of your kubeconfig file here. 17 | Kubeconfig: "", 18 | 19 | // The namespace must be created upfront. 20 | Namespace: "kube-system", 21 | 22 | // Name of helm release. 23 | Name: "coredns", 24 | 25 | // Repositories must be added upfront as well. 26 | Chart: "stable/coredns", 27 | 28 | // Values passed to the release in YAML format. 29 | Values: `replicas: 1 30 | labels: 31 | foo: bar 32 | `, 33 | // Version of the chart to use. 34 | Version: "1.12.0", 35 | } 36 | 37 | if _, err := config.New(); err == nil { 38 | t.Fatalf("Creating release object with bad kubeconfig should fail") 39 | } 40 | } 41 | 42 | func newConfig(t *testing.T) *release.Config { 43 | t.Helper() 44 | 45 | pki := &pki.PKI{ 46 | Certificate: pki.Certificate{ 47 | RSABits: 512, 48 | }, 49 | Kubernetes: &pki.Kubernetes{}, 50 | } 51 | 52 | if err := pki.Generate(); err != nil { 53 | t.Fatalf("Generating PKI: %v", err) 54 | } 55 | 56 | clientConfig := client.Config{ 57 | Server: "foo", 58 | CACertificate: pki.Kubernetes.CA.X509Certificate, 59 | ClientCertificate: pki.Kubernetes.AdminCertificate.X509Certificate, 60 | ClientKey: pki.Kubernetes.AdminCertificate.PrivateKey, 61 | } 62 | 63 | kubeconfig, err := clientConfig.ToYAMLString() 64 | if err != nil { 65 | t.Fatalf("Rendering kubeconfig: %v", err) 66 | } 67 | 68 | return &release.Config{ 69 | // Put content of your kubeconfig file here. 70 | Kubeconfig: kubeconfig, 71 | 72 | // The namespace must be created upfront. 73 | Namespace: "kube-system", 74 | 75 | // Name of helm release. 76 | Name: "coredns", 77 | 78 | // Repositories must be added upfront as well. 79 | Chart: "stable/asgasgasgsa", 80 | 81 | // Values passed to the release in YAML format. 82 | Values: `replicas: 1 83 | labels: 84 | foo: bar 85 | `, 86 | // Version of the chart to use. 87 | Version: "1.12.0", 88 | } 89 | } 90 | 91 | func newRelease(t *testing.T) release.Release { 92 | t.Helper() 93 | 94 | config := newConfig(t) 95 | 96 | r, err := config.New() 97 | if err != nil { 98 | t.Fatalf("Creating release object with valid kubeconfig should succeed: %v", err) 99 | } 100 | 101 | return r 102 | } 103 | 104 | //nolint:paralleltest // Helm client is not thread-safe. 105 | func TestConfigNew(t *testing.T) { 106 | newRelease(t) 107 | } 108 | 109 | // Validate() tests. 110 | // 111 | //nolint:paralleltest // Helm client is not thread-safe. 112 | func TestConfigValidateEmptyNamespace(t *testing.T) { 113 | c := newConfig(t) 114 | c.Namespace = "" 115 | 116 | if err := c.Validate(); err == nil { 117 | t.Fatalf("Validate should require namespace to be set") 118 | } 119 | } 120 | 121 | //nolint:paralleltest // Helm client is not thread-safe. 122 | func TestConfigValidateEmptyName(t *testing.T) { 123 | c := newConfig(t) 124 | c.Name = "" 125 | 126 | if err := c.Validate(); err == nil { 127 | t.Fatalf("Validate should require name to be set") 128 | } 129 | } 130 | 131 | //nolint:paralleltest // Helm client is not thread-safe. 132 | func TestConfigValidateEmptyChart(t *testing.T) { 133 | c := newConfig(t) 134 | c.Chart = "" 135 | 136 | if err := c.Validate(); err == nil { 137 | t.Fatalf("Validate should require chart to be set") 138 | } 139 | } 140 | 141 | //nolint:paralleltest // Helm client is not thread-safe. 142 | func TestConfigValidateBadValues(t *testing.T) { 143 | c := newConfig(t) 144 | c.Values = "asd" 145 | 146 | if err := c.Validate(); err == nil { 147 | t.Fatalf("Validate should validate given values") 148 | } 149 | } 150 | 151 | // ValidateChart() tests. 152 | // 153 | //nolint:paralleltest // Helm client is not thread-safe. 154 | func TestReleaseValidateChartBad(t *testing.T) { 155 | r := newRelease(t) 156 | 157 | if err := r.ValidateChart(); err == nil { 158 | t.Fatalf("Validating invalid chart should fail") 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /pkg/etcd/member_test.go: -------------------------------------------------------------------------------- 1 | package etcd_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/internal/utiltest" 7 | "github.com/flexkube/libflexkube/pkg/defaults" 8 | "github.com/flexkube/libflexkube/pkg/etcd" 9 | "github.com/flexkube/libflexkube/pkg/host" 10 | "github.com/flexkube/libflexkube/pkg/host/transport/direct" 11 | ) 12 | 13 | const ( 14 | nonEmptyString = "foo" 15 | ) 16 | 17 | func TestMemberToHostConfiguredContainer(t *testing.T) { 18 | t.Parallel() 19 | 20 | cert := utiltest.GenerateX509Certificate(t) 21 | privateKey := utiltest.GenerateRSAPrivateKey(t) 22 | 23 | kas := &etcd.MemberConfig{ 24 | Name: nonEmptyString, 25 | PeerAddress: nonEmptyString, 26 | CACertificate: cert, 27 | PeerCertificate: cert, 28 | PeerKey: privateKey, 29 | ServerCertificate: cert, 30 | ServerKey: privateKey, 31 | Image: defaults.EtcdImage, 32 | PeerCertAllowedCN: nonEmptyString, 33 | Host: host.Host{ 34 | DirectConfig: &direct.Config{}, 35 | }, 36 | } 37 | 38 | o, err := kas.New() 39 | if err != nil { 40 | t.Fatalf("New should not return error, got: %v", err) 41 | } 42 | 43 | hcc, err := o.ToHostConfiguredContainer() 44 | if err != nil { 45 | t.Fatalf("Generating HostConfiguredContainer should work, got: %v", err) 46 | } 47 | 48 | if _, err := hcc.New(); err != nil { 49 | t.Fatalf("ToHostConfiguredContainer() should generate valid HostConfiguredContainer, got: %v", err) 50 | } 51 | } 52 | 53 | func validMember(t *testing.T) *etcd.MemberConfig { 54 | t.Helper() 55 | 56 | cert := utiltest.GenerateX509Certificate(t) 57 | privateKey := utiltest.GenerateRSAPrivateKey(t) 58 | 59 | return &etcd.MemberConfig{ 60 | Name: nonEmptyString, 61 | PeerAddress: nonEmptyString, 62 | CACertificate: cert, 63 | PeerCertificate: cert, 64 | PeerKey: privateKey, 65 | ServerCertificate: cert, 66 | ServerKey: privateKey, 67 | Image: defaults.EtcdImage, 68 | PeerCertAllowedCN: nonEmptyString, 69 | Host: host.Host{ 70 | DirectConfig: &direct.Config{}, 71 | }, 72 | } 73 | } 74 | 75 | // Validate() tests. 76 | // 77 | //nolint:funlen // Just many test cases. 78 | func TestValidate(t *testing.T) { 79 | t.Parallel() 80 | 81 | cases := map[string]struct { 82 | mutator func(m *etcd.MemberConfig) *etcd.MemberConfig 83 | expectError bool 84 | }{ 85 | "valid": { 86 | func(m *etcd.MemberConfig) *etcd.MemberConfig { return m }, 87 | false, 88 | }, 89 | "peer address": { 90 | func(m *etcd.MemberConfig) *etcd.MemberConfig { 91 | m.PeerAddress = "" 92 | 93 | return m 94 | }, 95 | true, 96 | }, 97 | "member name": { 98 | func(m *etcd.MemberConfig) *etcd.MemberConfig { 99 | m.Name = "" 100 | 101 | return m 102 | }, 103 | true, 104 | }, 105 | "CA certificate": { 106 | func(m *etcd.MemberConfig) *etcd.MemberConfig { 107 | m.CACertificate = nonEmptyString 108 | 109 | return m 110 | }, 111 | true, 112 | }, 113 | "peer certificate": { 114 | func(m *etcd.MemberConfig) *etcd.MemberConfig { 115 | m.PeerCertificate = nonEmptyString 116 | 117 | return m 118 | }, 119 | true, 120 | }, 121 | "server certificate": { 122 | func(m *etcd.MemberConfig) *etcd.MemberConfig { 123 | m.ServerCertificate = nonEmptyString 124 | 125 | return m 126 | }, 127 | true, 128 | }, 129 | "peer key": { 130 | func(m *etcd.MemberConfig) *etcd.MemberConfig { 131 | m.PeerKey = nonEmptyString 132 | 133 | return m 134 | }, 135 | true, 136 | }, 137 | "server key": { 138 | func(m *etcd.MemberConfig) *etcd.MemberConfig { 139 | m.ServerKey = nonEmptyString 140 | 141 | return m 142 | }, 143 | true, 144 | }, 145 | "bad host": { 146 | func(m *etcd.MemberConfig) *etcd.MemberConfig { 147 | m.Host.DirectConfig = nil 148 | 149 | return m 150 | }, 151 | true, 152 | }, 153 | } 154 | 155 | for c, testCase := range cases { 156 | testCase := testCase 157 | 158 | t.Run(c, func(t *testing.T) { 159 | t.Parallel() 160 | 161 | m := testCase.mutator(validMember(t)) 162 | 163 | err := m.Validate() 164 | if testCase.expectError && err == nil { 165 | t.Fatalf("Expected error") 166 | } 167 | 168 | if !testCase.expectError && err != nil { 169 | t.Fatalf("Didn't expect error, got: %v", err) 170 | } 171 | }) 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /pkg/host/host.go: -------------------------------------------------------------------------------- 1 | // Package host collects all transport interface implementations and provides an 2 | // unified configuration interface for these. 3 | package host 4 | 5 | import ( 6 | "fmt" 7 | 8 | "github.com/flexkube/libflexkube/internal/util" 9 | "github.com/flexkube/libflexkube/pkg/host/transport" 10 | "github.com/flexkube/libflexkube/pkg/host/transport/direct" 11 | "github.com/flexkube/libflexkube/pkg/host/transport/ssh" 12 | ) 13 | 14 | // Host allows to forward TCP ports, UNIX sockets to local machine to establish 15 | // communication with remote daemons. 16 | // 17 | // Exactly one transport method must be configured. 18 | type Host struct { 19 | // DirectConfig indicates, that no forwarding should occur, so addresses will 20 | // be returned as given. 21 | DirectConfig *direct.Config `json:"direct,omitempty"` 22 | 23 | // SSHConfig configures given addresses to be forwarded using SSH tunneling. 24 | SSHConfig *ssh.Config `json:"ssh,omitempty"` 25 | } 26 | 27 | type host struct { 28 | transport transport.Interface 29 | } 30 | 31 | type hostConnected struct { 32 | transport transport.Connected 33 | } 34 | 35 | // New validates Host configuration and sets configured transport method. 36 | func (h *Host) New() (transport.Interface, error) { 37 | if err := h.Validate(); err != nil { 38 | return nil, fmt.Errorf("validating host configuration: %w", err) 39 | } 40 | 41 | // TODO that seems ugly, is there a better way to generalize it? 42 | var configuredTransport transport.Interface 43 | 44 | if h.DirectConfig != nil { 45 | configuredTransport, _ = h.DirectConfig.New() //nolint:errcheck // We check it in Validate(). 46 | } 47 | 48 | if h.SSHConfig != nil { 49 | configuredTransport, _ = h.SSHConfig.New() //nolint:errcheck // We check it in Validate(). 50 | } 51 | 52 | return &host{ 53 | transport: configuredTransport, 54 | }, nil 55 | } 56 | 57 | // Validate validates host configuration. 58 | func (h *Host) Validate() error { 59 | var errors util.ValidateErrors 60 | 61 | if err := h.DirectConfig.Validate(); err != nil { 62 | errors = append(errors, fmt.Errorf("validating direct config: %w", err)) 63 | } 64 | 65 | if h.DirectConfig != nil && h.SSHConfig != nil { 66 | errors = append(errors, fmt.Errorf("host must have only one transport method defined")) 67 | } 68 | 69 | if h.DirectConfig == nil && h.SSHConfig == nil { 70 | errors = append(errors, fmt.Errorf("host must have transport method defined")) 71 | } 72 | 73 | if h.SSHConfig != nil { 74 | if err := h.SSHConfig.Validate(); err != nil { 75 | errors = append(errors, fmt.Errorf("validating SSH config: %w", err)) 76 | } 77 | } 78 | 79 | return errors.Return() 80 | } 81 | 82 | // selectTransport returns transport protocol configured for container. 83 | // 84 | // It returns error if transport protocol configuration is invalid. 85 | func (h *host) Connect() (transport.Connected, error) { 86 | connected, err := h.transport.Connect() 87 | if err != nil { 88 | return nil, fmt.Errorf("connecting: %w", err) 89 | } 90 | 91 | return &hostConnected{ 92 | transport: connected, 93 | }, nil 94 | } 95 | 96 | // ForwardUnixSocket forwards given unix socket path using configured transport method and returns 97 | // local unix socket address. 98 | func (h *hostConnected) ForwardUnixSocket(path string) (string, error) { 99 | return h.transport.ForwardUnixSocket(path) 100 | } 101 | 102 | // ForwardTCP forwards given TCP address using configured transport method and returns local 103 | // address with port. 104 | func (h *hostConnected) ForwardTCP(address string) (string, error) { 105 | return h.transport.ForwardTCP(address) 106 | } 107 | 108 | // BuildConfig merges values from both host objects. This is a helper method used for building hierarchical 109 | // configuration. 110 | func BuildConfig(config, defaults Host) Host { 111 | // If config has no direct config configured or has SSH config configured, build SSH configuration. 112 | if (config.DirectConfig == nil && defaults.SSHConfig != nil) || config.SSHConfig != nil { 113 | config.SSHConfig = ssh.BuildConfig(config.SSHConfig, defaults.SSHConfig) 114 | } 115 | 116 | // If config has nothing configured and default has no SSH configuration configured, 117 | // return direct config as a default. 118 | if config.DirectConfig == nil && config.SSHConfig == nil && defaults.SSHConfig == nil { 119 | return Host{ 120 | DirectConfig: &direct.Config{}, 121 | } 122 | } 123 | 124 | return config 125 | } 126 | -------------------------------------------------------------------------------- /pkg/container/hostconfiguredcontainer_integration_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package container 5 | 6 | import ( 7 | "fmt" 8 | "path" 9 | "testing" 10 | "time" 11 | 12 | "github.com/flexkube/libflexkube/pkg/container/runtime/docker" 13 | "github.com/flexkube/libflexkube/pkg/container/types" 14 | "github.com/flexkube/libflexkube/pkg/host" 15 | "github.com/flexkube/libflexkube/pkg/host/transport/direct" 16 | ) 17 | 18 | const ( 19 | // Arbitrary value of wow long we wait for container to start and report as running by Docker. 20 | containerRunningDelay = 3 * time.Second 21 | ) 22 | 23 | // Create() tests. 24 | // 25 | //nolint:funlen // Just a long integration test. 26 | func TestHostConfiguredContainerDeployConfigFile(t *testing.T) { 27 | t.Parallel() 28 | 29 | basePath := "/tmp/foo" 30 | filePath := path.Join(basePath, randomContainerName(t)) 31 | 32 | hccConfig := &HostConfiguredContainer{ 33 | Host: host.Host{ 34 | DirectConfig: &direct.Config{}, 35 | }, 36 | Container: Container{ 37 | Runtime: RuntimeConfig{ 38 | Docker: &docker.Config{}, 39 | }, 40 | Config: types.ContainerConfig{ 41 | Name: randomContainerName(t), 42 | Image: "nginx", 43 | Mounts: []types.Mount{ 44 | { 45 | Source: fmt.Sprintf("%s/", basePath), 46 | Target: basePath, 47 | }, 48 | }, 49 | Entrypoint: []string{"/bin/sh"}, 50 | Args: []string{ 51 | "-c", 52 | fmt.Sprintf("grep baz %s && tail -f /dev/null", filePath), 53 | }, 54 | }, 55 | }, 56 | ConfigFiles: map[string]string{ 57 | filePath: "baz", 58 | }, 59 | } 60 | 61 | hcc, err := hccConfig.New() 62 | if err != nil { 63 | t.Fatalf("Initializing host configured container should succeed, got: %v", err) 64 | } 65 | 66 | if err = hcc.Configure([]string{filePath}); err != nil { 67 | t.Fatalf("Configuring host configured container should succeed, got: %v", err) 68 | } 69 | 70 | if err = hcc.Create(); err != nil { 71 | t.Fatalf("Creating host configured container should succeed, got: %v", err) 72 | } 73 | 74 | if err = hcc.Start(); err != nil { 75 | t.Fatalf("Starting host configured container should succeed, got: %v", err) 76 | } 77 | 78 | // Sleep a bit, to make sure container starts etc. 79 | time.Sleep(containerRunningDelay) 80 | 81 | if err = hcc.Status(); err != nil { 82 | t.Fatalf("Checking host configured container status should succeed, got: %v", err) 83 | } 84 | 85 | testHCC, ok := hcc.(*hostConfiguredContainer) 86 | if !ok { 87 | t.Fatalf("Unexpected type for host configured container: %T", hcc) 88 | } 89 | 90 | s := testHCC.container.Status().Status 91 | if s != "running" { 92 | t.Errorf("Host configured container should be running, got status %v", s) 93 | } 94 | 95 | if err = hcc.Stop(); err != nil { 96 | t.Errorf("Stopping host configured container status should succeed, got: %v", err) 97 | } 98 | 99 | if err = hcc.Delete(); err != nil { 100 | t.Fatalf("Deleting host configured container status should succeed, got: %v", err) 101 | } 102 | } 103 | 104 | func TestHostConfiguredContainerPostStartHook(t *testing.T) { 105 | t.Parallel() 106 | 107 | hookCalled := false 108 | 109 | hookF := Hook(func() error { 110 | hookCalled = true 111 | 112 | return nil 113 | }) 114 | 115 | hccConfig := &HostConfiguredContainer{ 116 | Host: host.Host{ 117 | DirectConfig: &direct.Config{}, 118 | }, 119 | Container: Container{ 120 | Runtime: RuntimeConfig{ 121 | Docker: &docker.Config{}, 122 | }, 123 | Config: types.ContainerConfig{ 124 | Name: "foo", 125 | Image: "busybox:latest", 126 | }, 127 | }, 128 | Hooks: &Hooks{ 129 | PostStart: &hookF, 130 | }, 131 | } 132 | 133 | hcc, err := hccConfig.New() 134 | if err != nil { 135 | t.Fatalf("Initializing host configured container should succeed, got: %v", err) 136 | } 137 | 138 | if err = hcc.Create(); err != nil { 139 | t.Fatalf("Creating host configured container should succeed, got: %v", err) 140 | } 141 | 142 | if err = hcc.Start(); err != nil { 143 | t.Fatalf("Starting host configured container should succeed, got: %v", err) 144 | } 145 | 146 | if !hookCalled { 147 | t.Errorf("PostStart hook should be called") 148 | } 149 | 150 | if err = hcc.Stop(); err != nil { 151 | t.Errorf("Stopping host configured container status should succeed, got: %v", err) 152 | } 153 | 154 | if err = hcc.Delete(); err != nil { 155 | t.Fatalf("Deleting host configured container status should succeed, got: %v", err) 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /pkg/pki/etcd_test.go: -------------------------------------------------------------------------------- 1 | package pki_test 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/google/go-cmp/cmp" 8 | 9 | "github.com/flexkube/libflexkube/pkg/pki" 10 | ) 11 | 12 | func TestGenerateEtcdPeerCertificates(t *testing.T) { 13 | t.Parallel() 14 | 15 | pki := &pki.PKI{ 16 | Etcd: &pki.Etcd{ 17 | PeerCertificates: map[string]*pki.Certificate{ 18 | "foo": { 19 | Organization: "foo", 20 | }, 21 | }, 22 | }, 23 | } 24 | 25 | if err := pki.Generate(); err != nil { 26 | t.Fatalf("Generating valid PKI should work, got: %v", err) 27 | } 28 | 29 | if pki.Etcd.PeerCertificates["foo"].X509Certificate == "" { 30 | t.Fatalf("Generated etcd peer certificate should not be empty") 31 | } 32 | } 33 | 34 | func TestGenerateEtcdPeerCertificatesPropagate(t *testing.T) { 35 | t.Parallel() 36 | 37 | expectedIPs := []net.IP{net.ParseIP("1.1.1.1"), net.ParseIP("127.0.0.1")} 38 | 39 | pki := &pki.PKI{ 40 | Etcd: &pki.Etcd{ 41 | Peers: map[string]string{ 42 | "foo": "1.1.1.1", 43 | }, 44 | PeerCertificates: map[string]*pki.Certificate{ 45 | "foo": { 46 | Organization: "foo", 47 | }, 48 | }, 49 | }, 50 | } 51 | 52 | if err := pki.Generate(); err != nil { 53 | t.Fatalf("Generating valid PKI should work, got: %v", err) 54 | } 55 | 56 | if pki.Etcd.PeerCertificates["foo"].X509Certificate == "" { 57 | t.Fatalf("Generated etcd peer certificate should not be empty") 58 | } 59 | 60 | c, err := pki.Etcd.PeerCertificates["foo"].DecodeX509Certificate() 61 | if err != nil { 62 | t.Fatalf("Decoding generated certificate should work, got: %v", err) 63 | } 64 | 65 | if diff := cmp.Diff(c.IPAddresses, expectedIPs); diff != "" { 66 | t.Fatalf("Unexpected diff: %s", diff) 67 | } 68 | } 69 | 70 | func TestGenerateEtcdPeerCertitificatesSupportAddingPeers(t *testing.T) { 71 | t.Parallel() 72 | 73 | pki := &pki.PKI{ 74 | Etcd: &pki.Etcd{ 75 | Peers: map[string]string{ 76 | "foo": "1.1.1.1", 77 | }, 78 | PeerCertificates: map[string]*pki.Certificate{ 79 | "foo": { 80 | Organization: "foo", 81 | }, 82 | }, 83 | }, 84 | } 85 | 86 | if err := pki.Generate(); err != nil { 87 | t.Fatalf("Generating valid PKI should work, got: %v", err) 88 | } 89 | 90 | pki.Etcd.Peers["bar"] = "2.2.2.2" 91 | 92 | if err := pki.Generate(); err != nil { 93 | t.Fatalf("Generating valid PKI should work, got: %v", err) 94 | } 95 | 96 | if pki.Etcd.PeerCertificates["bar"].X509Certificate == "" { 97 | t.Fatalf("Generated etcd peer certificate should not be empty") 98 | } 99 | } 100 | 101 | func TestGenerateEtcdPeerCertitificatesPreservePeers(t *testing.T) { 102 | t.Parallel() 103 | 104 | pki := &pki.PKI{ 105 | Etcd: &pki.Etcd{ 106 | Peers: map[string]string{ 107 | "foo": "1.1.1.1", 108 | }, 109 | }, 110 | } 111 | 112 | if err := pki.Generate(); err != nil { 113 | t.Fatalf("Generating valid PKI should work, got: %v", err) 114 | } 115 | 116 | pki.Etcd.Peers = map[string]string{} 117 | 118 | if err := pki.Generate(); err != nil { 119 | t.Fatalf("Generating valid PKI should work, got: %v", err) 120 | } 121 | 122 | if pki.Etcd.PeerCertificates["foo"].X509Certificate == "" { 123 | t.Fatalf("Generated etcd peer certificate should not be empty") 124 | } 125 | } 126 | 127 | func TestGenerateEtcdPeerCertitificatesAddServer(t *testing.T) { 128 | t.Parallel() 129 | 130 | pki := &pki.PKI{ 131 | Etcd: &pki.Etcd{ 132 | Peers: map[string]string{ 133 | "foo": "1.1.1.1", 134 | }, 135 | }, 136 | } 137 | 138 | if err := pki.Generate(); err != nil { 139 | t.Fatalf("Generating valid PKI should work, got: %v", err) 140 | } 141 | 142 | pki.Etcd.Servers = map[string]string{"bar": "2.2.2.2"} 143 | 144 | if err := pki.Generate(); err != nil { 145 | t.Fatalf("Generating valid PKI should work, got: %v", err) 146 | } 147 | 148 | if pki.Etcd.ServerCertificates["bar"].X509Certificate == "" { 149 | t.Fatalf("Generated etcd server certificate should not be empty") 150 | } 151 | } 152 | 153 | func TestGenerateEtcdPeerCertificatesDontSetCommonName(t *testing.T) { 154 | t.Parallel() 155 | 156 | pki := &pki.PKI{ 157 | Etcd: &pki.Etcd{ 158 | Peers: map[string]string{ 159 | "foo": "1.1.1.1", 160 | }, 161 | }, 162 | } 163 | 164 | if err := pki.Generate(); err != nil { 165 | t.Fatalf("Generating valid PKI should work, got: %v", err) 166 | } 167 | 168 | if err := pki.Generate(); err != nil { 169 | t.Fatalf("Generating valid PKI should work, got: %v", err) 170 | } 171 | 172 | if pki.Etcd.PeerCertificates["foo"].CommonName != "" { 173 | t.Fatalf("Generated etcd peer certificate should have empty common name") 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /pkg/container/types/types.go: -------------------------------------------------------------------------------- 1 | // Package types contains types used for managing the containers. They are put in separate package 2 | // to avoid cyclic dependencies while importing. 3 | package types 4 | 5 | // ContainerConfig stores runtime-agnostic information how to run the container. 6 | type ContainerConfig struct { 7 | // Name is a name of the container. 8 | Name string `json:"name"` 9 | 10 | // Image is a container image to use. 11 | Image string `json:"image"` 12 | 13 | // Args is a list of arguments to pass to the container. 14 | Args []string `json:"args,omitempty"` 15 | 16 | // Entrypoint is a binary, which will be started in the container. 17 | Entrypoint []string `json:"entrypoint,omitempty"` 18 | 19 | // Ports is a list of ports, which will be exposed by the container. 20 | Ports []PortMap `json:"ports,omitempty"` 21 | 22 | // Mounts is a list of mounts, which will be added to the container. 23 | Mounts []Mount `json:"mounts,omitempty"` 24 | 25 | // Privileged controls, if created container should have full access to the 26 | // host. 27 | Privileged bool `json:"privileged,omitempty"` 28 | 29 | // NetworkMode defines what network the container should use. 30 | // 31 | // Valid values depends on used container runtime. 32 | NetworkMode string `json:"networkMode,omitempty"` 33 | 34 | // PidMode defines, in which PID namespace container should run. 35 | // 36 | // Valid values depends on used container runtime. 37 | PidMode string `json:"pidMode,omitempty"` 38 | 39 | // IpcMode defines, in which IPC namespace container should run. 40 | // 41 | // Valid values depends on used container runtime. 42 | IpcMode string `json:"ipcMode,omitempty"` 43 | 44 | // User defines, as which user the container should run. 45 | User string `json:"user,omitempty"` 46 | 47 | // Group defines as which group the container should run. 48 | Group string `json:"group,omitempty"` 49 | 50 | // Env defines a key-value environment variables to set in the container. 51 | Env map[string]string `json:"env,omitempty"` 52 | } 53 | 54 | // ContainerStatus stores status information received from the runtime. 55 | // 56 | // TODO: This should cover all fields which are defined in ContainerConfig, 57 | // so we can read and compare if actual configuration matches our expectations. 58 | type ContainerStatus struct { 59 | // ID is a runtime specific container ID. 60 | ID string `json:"id,omitempty"` 61 | 62 | // Status is a runtime specific status string. 63 | Status string `json:"status,omitempty"` 64 | } 65 | 66 | // PortMap is basically a github.com/docker/go-connections/nat.PortMap. 67 | // 68 | // TODO: Once we introduce Kubelet runtime, we need to figure out how to structure it. 69 | type PortMap struct { 70 | // IP is an IP address on which container port should be exposed. 71 | IP string `json:"ip"` 72 | 73 | // Port defines, which port should be exposed. 74 | Port int `json:"port"` 75 | 76 | // Protocol defines what protocol should be exposed from the container. 77 | Protocol string `json:"protocol"` 78 | } 79 | 80 | // Mount describe host bind mount. 81 | // 82 | // TODO: Same as PortMap. 83 | type Mount struct { 84 | // Source is a host filesystem path which will be mounted into the container. 85 | // 86 | // Example value: '/opt/bin'. 87 | Source string `json:"source"` 88 | 89 | // Target is a path in container's filesystem where host path will be mounted. 90 | // 91 | // Example value: '/usr/local/bin'. 92 | Target string `json:"target"` 93 | 94 | // Propagation defines how the mounts in host path will be propatated. 95 | // 96 | // Valid value depends on used container runtime. 97 | Propagation string `json:"propagation,omitempty"` 98 | } 99 | 100 | // File describes file, which can be either copied to or from container. 101 | type File struct { 102 | // Path is a path on the filesystem. 103 | Path string `json:"path"` 104 | 105 | // Content is a content of the file. Binary files are currently not supported. 106 | Content string `json:"content"` 107 | 108 | // Mode is a numeric file mode. 109 | Mode int64 `json:"mode"` 110 | 111 | // User is an owner of the file. 112 | User string `json:"uid"` 113 | 114 | // Group is a group owner of the file. 115 | Group string `json:"gid"` 116 | } 117 | 118 | // Exists controls, how container existence is determined based on ContainerStatus. 119 | // If state has no ID set, it means that container does not exist. 120 | func (s *ContainerStatus) Exists() bool { 121 | return s.ID != "" 122 | } 123 | 124 | // Running determines if container is running, based on ContainerStatus. 125 | func (s *ContainerStatus) Running() bool { 126 | return s.Exists() && s.Status == "running" 127 | } 128 | 129 | // Restarting returns true, if container is restarting in a loop, based on ContainerStatus. 130 | func (s *ContainerStatus) Restarting() bool { 131 | return s.Exists() && s.Status == "restarting" 132 | } 133 | -------------------------------------------------------------------------------- /pkg/container/containersstate.go: -------------------------------------------------------------------------------- 1 | package container 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/flexkube/libflexkube/pkg/container/runtime/docker" 7 | "github.com/flexkube/libflexkube/pkg/container/types" 8 | ) 9 | 10 | const ( 11 | // StatusMissing is a value, which is set to ContainerStatus.Status field, 12 | // if stored container ID is not found. 13 | StatusMissing = "gone" 14 | ) 15 | 16 | // ContainersStateInterface represents 'constainersState' capabilities. 17 | type ContainersStateInterface interface { 18 | // CheckState updates the state of all previously configured containers 19 | // and their configuration on the host 20 | CheckState() error 21 | 22 | // RemoveContainer removes the container by ID. 23 | RemoveContainer(containerName string) error 24 | 25 | // CreateAndStart is a helper, which creates and spawns given container. 26 | CreateAndStart(containerName string) error 27 | 28 | // Export converts unexported containersState to exported type, so it can be serialized and stored. 29 | Export() ContainersState 30 | } 31 | 32 | // ContainersState represents states of multiple containers. 33 | type ContainersState map[string]*HostConfiguredContainer 34 | 35 | // containerState is a validated version of ContainersState, which can be used to perform operations. 36 | type containersState map[string]*hostConfiguredContainer 37 | 38 | // New validates ContainersState struct and returns operational containerState. 39 | func (s ContainersState) New() (ContainersStateInterface, error) { 40 | if s == nil { 41 | s = ContainersState{} 42 | } 43 | 44 | state := containersState{} 45 | 46 | for name, container := range s { 47 | m, err := container.New() 48 | if err != nil { 49 | return nil, fmt.Errorf("creating new container object: %w", err) 50 | } 51 | 52 | hcc, ok := m.(*hostConfiguredContainer) 53 | if !ok { 54 | return nil, fmt.Errorf("converting container to internal version") 55 | } 56 | 57 | state[name] = hcc 58 | } 59 | 60 | return state, nil 61 | } 62 | 63 | // CheckState updates the state of all previously configured containers 64 | // and their configuration on the host. 65 | func (s containersState) CheckState() error { 66 | for containerName, hcc := range s { 67 | if err := hcc.Status(); err != nil { 68 | hcc.container.SetStatus(types.ContainerStatus{ 69 | Status: err.Error(), 70 | }) 71 | 72 | continue 73 | } 74 | 75 | if !hcc.container.Status().Exists() { 76 | hcc.container.SetStatus(types.ContainerStatus{ 77 | Status: StatusMissing, 78 | }) 79 | } 80 | 81 | if err := hcc.ConfigurationStatus(); err != nil { 82 | return fmt.Errorf("checking container %q configuration status: %w", containerName, err) 83 | } 84 | } 85 | 86 | return nil 87 | } 88 | 89 | // RemoveContainer removes the container by ID. 90 | func (s containersState) RemoveContainer(containerName string) error { 91 | if _, exists := s[containerName]; !exists { 92 | return fmt.Errorf("can't remove non-existing container") 93 | } 94 | 95 | status := s[containerName].container.Status() 96 | 97 | if status.Exists() && (status.Running() || status.Restarting()) { 98 | if err := s[containerName].Stop(); err != nil { 99 | return fmt.Errorf("stopping container before removing: %w", err) 100 | } 101 | } 102 | 103 | if status.Exists() { 104 | if err := s[containerName].Delete(); err != nil { 105 | return fmt.Errorf("removing container: %w", err) 106 | } 107 | } 108 | 109 | delete(s, containerName) 110 | 111 | return nil 112 | } 113 | 114 | // CreateAndStart is a helper, which creates and spawns given container. 115 | func (s containersState) CreateAndStart(containerName string) error { 116 | if _, exists := s[containerName]; !exists { 117 | return fmt.Errorf("can't create non-existing container") 118 | } 119 | 120 | if err := s[containerName].Create(); err != nil { 121 | return fmt.Errorf("creating new container: %w", err) 122 | } 123 | 124 | if err := s[containerName].Start(); err != nil { 125 | return fmt.Errorf("starting container: %w", err) 126 | } 127 | 128 | return nil 129 | } 130 | 131 | // Export converts unexported containersState to exported type, so it can be serialized and stored. 132 | func (s containersState) Export() ContainersState { 133 | exportedState := ContainersState{} 134 | 135 | for containerName, hcc := range s { 136 | exportedHCC := &HostConfiguredContainer{ //nolint:forcetypeassert // False positive?. 137 | Container: Container{ 138 | Config: hcc.container.Config(), 139 | Runtime: RuntimeConfig{ 140 | Docker: hcc.container.RuntimeConfig().(*docker.Config), 141 | }, 142 | }, 143 | Host: hcc.host, 144 | ConfigFiles: hcc.configFiles, 145 | } 146 | 147 | if s := hcc.container.Status(); s.ID != "" || s.Status != "" { 148 | exportedHCC.Container.Status = s 149 | } 150 | 151 | if exportedHCC.ConfigFiles == nil { 152 | exportedHCC.ConfigFiles = map[string]string{} 153 | } 154 | 155 | exportedState[containerName] = exportedHCC 156 | } 157 | 158 | return exportedState 159 | } 160 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | output: 2 | sort-results: true 3 | 4 | run: 5 | timeout: 10m 6 | build-tags: 7 | - e2e 8 | - integration 9 | 10 | issues: 11 | exclude-use-default: false 12 | max-same-issues: 0 13 | max-issues-per-linter: 0 14 | exclude-rules: 15 | # False positive: https://github.com/kunwardeep/paralleltest/issues/8. 16 | - linters: 17 | - paralleltest 18 | text: "does not use range value in test Run" 19 | - linters: 20 | - godot 21 | source: "(front proxy CA certificate,|certificate, as recommended by)" 22 | # Allow packages to print output to the user until we introduce loggers. 23 | - linters: 24 | - forbidigo 25 | text: "use of `fmt.Printf` forbidden by pattern" 26 | - linters: 27 | - forbidigo 28 | text: "use of `fmt.Println` forbidden by pattern" 29 | # Allow some errors to be not wrapped, as it is not ambiguous where they come from. 30 | - linters: 31 | - wrapcheck 32 | text: "error returned from external package is unwrapped" 33 | # Allow wrapping function which return errors. 34 | - linters: 35 | - wrapcheck 36 | text: "error returned from interface method should be wrapped" 37 | # Allow certain abbreviations in JSON fields for better readability. 38 | - linters: 39 | - tagliatelle 40 | text: "(CIDR|CA|DNS|API|CN|SSH).+(Cidr|Ca|Dns|Api|Cn|Ssh)" 41 | # Allow shadowing 'err' variables as this is convention. Other shadowing should still be avoided. 42 | - linters: 43 | - govet 44 | text: "shadow: declaration of \"err\" shadows declaration at line" 45 | # Allow using snake case for test and example names for better readability, as those functions are not called 46 | # directly by anyone anyway. 47 | - linters: 48 | - nosnakecase 49 | text: "^(Test_|Example)" 50 | 51 | linters-settings: 52 | errcheck: 53 | check-type-assertions: true 54 | check-blank: true 55 | gci: 56 | sections: 57 | - standard 58 | - default 59 | - prefix(github.com/flexkube/libflexkube) 60 | godot: 61 | capital: true 62 | gofumpt: 63 | extra-rules: true 64 | govet: 65 | enable-all: true 66 | disable: 67 | - fieldalignment 68 | nolintlint: 69 | allow-leading-space: false 70 | require-explanation: true 71 | require-specific: true 72 | wsl: 73 | force-err-cuddling: true 74 | 75 | linters: 76 | disable: 77 | # We are not that strict with imports to have a whitelist. 78 | - depguard 79 | # To be able to make use of Go zero-value feature. 80 | - exhaustruct 81 | # Triggers on TODOs in the code, which is fine to put. 82 | - godox 83 | # Disabled until we implement some error types and migrate to use them. 84 | - err113 85 | # We don't really care about sprintf performance. 86 | - perfsprint 87 | # To allow defensive approach when initializing structs. 88 | - ireturn 89 | # Disabled until tests are split and moved to the right file names. 90 | - testpackage 91 | # This linter has been deprecated. 92 | - copyloopvar 93 | - intrange 94 | enable: 95 | - asasalint 96 | - asciicheck 97 | - bidichk 98 | - bodyclose 99 | - canonicalheader 100 | - containedctx 101 | - contextcheck 102 | - cyclop 103 | - decorder 104 | - dogsled 105 | - dupl 106 | - dupword 107 | - durationcheck 108 | - errcheck 109 | - errchkjson 110 | - errname 111 | - errorlint 112 | - exhaustive 113 | - fatcontext 114 | - forbidigo 115 | - forcetypeassert 116 | - funlen 117 | - gci 118 | - ginkgolinter 119 | - gocheckcompilerdirectives 120 | - gochecknoglobals 121 | - gochecknoinits 122 | - gochecksumtype 123 | - gocognit 124 | - goconst 125 | - gocritic 126 | - gocyclo 127 | - godot 128 | - gofmt 129 | - gofumpt 130 | - goheader 131 | - goimports 132 | - gomoddirectives 133 | - gomodguard 134 | - goprintffuncname 135 | - gosec 136 | - gosimple 137 | - gosmopolitan 138 | - govet 139 | - grouper 140 | - importas 141 | - inamedparam 142 | - ineffassign 143 | - interfacebloat 144 | - lll 145 | - loggercheck 146 | - maintidx 147 | - makezero 148 | - mirror 149 | - misspell 150 | - mnd 151 | - musttag 152 | - nakedret 153 | - nestif 154 | - nilerr 155 | - nilnil 156 | - nlreturn 157 | - noctx 158 | - nolintlint 159 | - nonamedreturns 160 | - nosprintfhostport 161 | - paralleltest 162 | - prealloc 163 | - predeclared 164 | - promlinter 165 | - protogetter 166 | - reassign 167 | - revive 168 | - rowserrcheck 169 | - sloglint 170 | - spancheck 171 | - sqlclosecheck 172 | - staticcheck 173 | - stylecheck 174 | - tagalign 175 | - tagliatelle 176 | - tenv 177 | - testableexamples 178 | - testifylint 179 | - thelper 180 | - tparallel 181 | - unconvert 182 | - unparam 183 | - unused 184 | - usestdlibvars 185 | - varnamelen 186 | - wastedassign 187 | - whitespace 188 | - wrapcheck 189 | - wsl 190 | - zerologlint 191 | -------------------------------------------------------------------------------- /pkg/controlplane/kube-scheduler.go: -------------------------------------------------------------------------------- 1 | package controlplane 2 | 3 | import ( 4 | "fmt" 5 | 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | componentbaseconfig "k8s.io/component-base/config/v1alpha1" 8 | kubeschedulerconfig "k8s.io/kube-scheduler/config/v1" 9 | "sigs.k8s.io/yaml" 10 | 11 | "github.com/flexkube/libflexkube/internal/util" 12 | "github.com/flexkube/libflexkube/pkg/container" 13 | "github.com/flexkube/libflexkube/pkg/container/runtime/docker" 14 | containertypes "github.com/flexkube/libflexkube/pkg/container/types" 15 | "github.com/flexkube/libflexkube/pkg/defaults" 16 | "github.com/flexkube/libflexkube/pkg/host" 17 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 18 | ) 19 | 20 | // KubeScheduler represents kube-scheduler configuration data. 21 | type KubeScheduler struct { 22 | // Common stores common information between all controlplane components. 23 | Common *Common `json:"common,omitempty"` 24 | 25 | // Host defines on which host kube-scheduler container should be created. 26 | Host *host.Host `json:"host,omitempty"` 27 | 28 | // Kubeconfig stores client information used by kube-scheduler to talk to 29 | // Kubernetes API. 30 | Kubeconfig client.Config `json:"kubeconfig"` 31 | } 32 | 33 | // kubeScheduler is validated and usable version of KubeScheduler. 34 | type kubeScheduler struct { 35 | common Common 36 | host host.Host 37 | kubeconfig string 38 | } 39 | 40 | // ToHostConfiguredContainer converts kubeScheduler into generic container struct. 41 | func (k *kubeScheduler) ToHostConfiguredContainer() (*container.HostConfiguredContainer, error) { 42 | configFiles := map[string]string{} 43 | // TODO put all those path in a single place. Perhaps make them configurable with defaults too 44 | configFiles["/etc/kubernetes/kube-scheduler/kubeconfig"] = k.kubeconfig 45 | configFiles["/etc/kubernetes/kube-scheduler/pki/ca.crt"] = string(k.common.KubernetesCACertificate) 46 | configFiles["/etc/kubernetes/kube-scheduler/pki/front-proxy-ca.crt"] = string(k.common.FrontProxyCACertificate) 47 | 48 | config := &kubeschedulerconfig.KubeSchedulerConfiguration{ 49 | TypeMeta: metav1.TypeMeta{ 50 | Kind: "KubeSchedulerConfiguration", 51 | APIVersion: kubeschedulerconfig.SchemeGroupVersion.String(), 52 | }, 53 | ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ 54 | Kubeconfig: "/etc/kubernetes/kubeconfig", 55 | }, 56 | } 57 | 58 | configRaw, err := yaml.Marshal(config) 59 | if err != nil { 60 | return nil, fmt.Errorf("marshaling configuration: %w", err) 61 | } 62 | 63 | configFiles["/etc/kubernetes/kube-scheduler/kube-scheduler.yaml"] = string(configRaw) 64 | 65 | containerConfig := container.Container{ 66 | // TODO: This is weird. This sets docker as default runtime config. 67 | Runtime: container.RuntimeConfig{ 68 | Docker: docker.DefaultConfig(), 69 | }, 70 | Config: containertypes.ContainerConfig{ 71 | Name: "kube-scheduler", 72 | Image: util.PickString(k.common.Image, defaults.KubeSchedulerImage), 73 | Mounts: []containertypes.Mount{ 74 | { 75 | Source: "/etc/kubernetes/kube-scheduler/", 76 | Target: "/etc/kubernetes", 77 | }, 78 | }, 79 | Args: []string{ 80 | "kube-scheduler", 81 | // Load configuration from the config file. 82 | "--config=/etc/kubernetes/kube-scheduler.yaml", 83 | // Those additional kubeconfig files are suppose to be used with delegated kube-apiserver, 84 | // so scenarios, where there is more than one kube-apiserver and they differ in privilege level. 85 | // However, not specifying them results in ugly log messages, so we just specify them to create less 86 | // environmental noise. 87 | "--authentication-kubeconfig=/etc/kubernetes/kubeconfig", 88 | "--authorization-kubeconfig=/etc/kubernetes/kubeconfig", 89 | // From k8s 1.17.x, without specifying those flags, there are some warning log messages printed. 90 | "--requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt", 91 | "--client-ca-file=/etc/kubernetes/pki/ca.crt", 92 | }, 93 | }, 94 | } 95 | 96 | return &container.HostConfiguredContainer{ 97 | Host: k.host, 98 | ConfigFiles: configFiles, 99 | Container: containerConfig, 100 | }, nil 101 | } 102 | 103 | // New validates KubeScheduler struct and returns it's usable version. 104 | func (k *KubeScheduler) New() (container.ResourceInstance, error) { 105 | if k.Common == nil { 106 | k.Common = &Common{} 107 | } 108 | 109 | if k.Host == nil { 110 | k.Host = &host.Host{} 111 | } 112 | 113 | if err := k.Validate(); err != nil { 114 | return nil, fmt.Errorf("validating Kubernetes Scheduler configuration: %w", err) 115 | } 116 | 117 | kubeconfig, _ := k.Kubeconfig.ToYAMLString() //nolint:errcheck // We check it in Validate(). 118 | 119 | return &kubeScheduler{ 120 | common: *k.Common, 121 | host: *k.Host, 122 | kubeconfig: kubeconfig, 123 | }, nil 124 | } 125 | 126 | // Validate validates kube-scheduler configuration. 127 | func (k *KubeScheduler) Validate() error { 128 | schedulerValidator := validator{ 129 | Common: k.Common, 130 | Host: k.Host, 131 | Kubeconfig: k.Kubeconfig, 132 | YAML: k, 133 | } 134 | 135 | return schedulerValidator.validate(true) 136 | } 137 | -------------------------------------------------------------------------------- /pkg/kubernetes/client/kubeconfig.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "fmt" 7 | "text/template" 8 | 9 | "sigs.k8s.io/yaml" 10 | 11 | "github.com/flexkube/libflexkube/internal/util" 12 | "github.com/flexkube/libflexkube/pkg/types" 13 | ) 14 | 15 | // Config is a simplified version of kubeconfig. 16 | type Config struct { 17 | // Server is a Kubernetes API server address. 18 | // 19 | // Example value: 'k8s.example.com:6443'. 20 | // 21 | // This field is required. 22 | Server string `json:"server,omitempty"` 23 | 24 | // CACertificate stores PEM encoded X.509 CA certificate, which was used 25 | // to sign Kubernetes API server certificate. 26 | // 27 | // This field is required. 28 | CACertificate types.Certificate `json:"caCertificate,omitempty"` 29 | 30 | // ClientCertificate stores PEM encoded X.509 client certificate, which will 31 | // be used for authentication and authorization to Kubernetes API server. 32 | // 33 | // This field is optional if Token field is populated. 34 | ClientCertificate types.Certificate `json:"clientCertificate,omitempty"` 35 | 36 | // ClientCertificate stores PEM encoded private key in PKCS1, PKCS8 or EC formats, 37 | // which will be used for authentication and authorization to Kubernetes API server. 38 | // Key must match configured ClientCertificate. 39 | // 40 | // This field is optional if Token field is populated. 41 | ClientKey types.PrivateKey `json:"clientKey,omitempty"` 42 | 43 | // Token stores Kubernetes token, which will be used for authentication and authrization 44 | // to Kubernetes API server. Usually used by kubelet to perform TLS bootstrapping. 45 | Token string `json:"token,omitempty"` 46 | } 47 | 48 | // Validate validates Config struct. 49 | func (c *Config) Validate() error { 50 | var errors util.ValidateErrors 51 | 52 | if c.Server == "" { 53 | errors = append(errors, fmt.Errorf("server is empty")) 54 | } 55 | 56 | if c.CACertificate == "" { 57 | errors = append(errors, fmt.Errorf("ca certificate is empty")) 58 | } 59 | 60 | errors = append(errors, c.validateAuth()...) 61 | 62 | b, err := yaml.Marshal(c) 63 | if err != nil { 64 | return append(errors, fmt.Errorf("marshaling config: %w", err)) 65 | } 66 | 67 | if err := yaml.Unmarshal(b, c); err != nil { 68 | return append(errors, fmt.Errorf("unmarshaling config: %w", err)) 69 | } 70 | 71 | return errors.Return() 72 | } 73 | 74 | func (c *Config) validateAuth() util.ValidateErrors { 75 | var errors util.ValidateErrors 76 | 77 | if c.ClientCertificate == "" && c.Token == "" { 78 | errors = append(errors, fmt.Errorf("either client certificate or token must be set")) 79 | } 80 | 81 | if c.ClientKey == "" && c.Token == "" { 82 | errors = append(errors, fmt.Errorf("either client key or token must be set")) 83 | } 84 | 85 | if c.Token != "" && c.ClientCertificate != "" { 86 | errors = append(errors, fmt.Errorf("client certificate should not be set together with token")) 87 | } 88 | 89 | if c.Token != "" && c.ClientKey != "" { 90 | errors = append(errors, fmt.Errorf("client key should not be set together with token")) 91 | } 92 | 93 | return errors 94 | } 95 | 96 | // ToYAMLString converts given configuration to kubeconfig format as YAML text. 97 | func (c *Config) ToYAMLString() (string, error) { 98 | if err := c.Validate(); err != nil { 99 | return "", fmt.Errorf("validating config: %w", err) 100 | } 101 | 102 | kubeconfig, err := c.renderKubeconfig() 103 | if err != nil { 104 | return "", fmt.Errorf("rendering kubeconfig: %w", err) 105 | } 106 | 107 | // Parse generated kubeconfig with Kubernetes client, to make sure everything is correct. 108 | if _, err := NewClient([]byte(kubeconfig)); err != nil { 109 | return "", fmt.Errorf("generated kubeconfig is invalid: %w", err) 110 | } 111 | 112 | return kubeconfig, nil 113 | } 114 | 115 | // renderKubeconfig renders Config as kubeconfig YAML. 116 | func (c *Config) renderKubeconfig() (string, error) { 117 | kubeconfigTemplate := `apiVersion: v1 118 | kind: Config 119 | clusters: 120 | - name: static 121 | cluster: 122 | server: https://{{ .Server }} 123 | certificate-authority-data: {{ .CACertificate }} 124 | users: 125 | - name: static 126 | user: 127 | {{- if .ClientCertificate }} 128 | client-certificate-data: {{ .ClientCertificate }} 129 | {{- end }} 130 | {{- if .ClientKey }} 131 | client-key-data: {{ .ClientKey }} 132 | {{- end }} 133 | {{- if .Token }} 134 | token: {{ .Token }} 135 | {{- end }} 136 | current-context: static 137 | contexts: 138 | - name: static 139 | context: 140 | cluster: static 141 | user: static 142 | ` 143 | 144 | data := struct { 145 | Server string 146 | CACertificate string 147 | ClientCertificate string 148 | ClientKey string 149 | Token string 150 | }{ 151 | c.Server, 152 | base64.StdEncoding.EncodeToString([]byte(c.CACertificate)), 153 | base64.StdEncoding.EncodeToString([]byte(c.ClientCertificate)), 154 | base64.StdEncoding.EncodeToString([]byte(c.ClientKey)), 155 | c.Token, 156 | } 157 | 158 | var buf bytes.Buffer 159 | 160 | tpl := template.Must(template.New("t").Parse(kubeconfigTemplate)) 161 | 162 | if err := tpl.Execute(&buf, data); err != nil { 163 | return "", fmt.Errorf("executing template: %w", err) 164 | } 165 | 166 | return buf.String(), nil 167 | } 168 | -------------------------------------------------------------------------------- /pkg/host/host_test.go: -------------------------------------------------------------------------------- 1 | package host 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | "github.com/flexkube/libflexkube/pkg/host/transport/direct" 8 | "github.com/flexkube/libflexkube/pkg/host/transport/ssh" 9 | ) 10 | 11 | // New() tests. 12 | func TestNew(t *testing.T) { 13 | t.Parallel() 14 | 15 | testHostConfig := BuildConfig(Host{ 16 | SSHConfig: &ssh.Config{ 17 | Address: "localhost", 18 | Password: "foo", 19 | }, 20 | }, Host{}) 21 | 22 | if _, err := testHostConfig.New(); err != nil { 23 | t.Fatalf("Built config should be valid, got: %v", err) 24 | } 25 | } 26 | 27 | func TestNewValidate(t *testing.T) { 28 | t.Parallel() 29 | 30 | h := &Host{} 31 | 32 | if _, err := h.New(); err == nil { 33 | t.Fatalf("New should validate the configuration") 34 | } 35 | } 36 | 37 | // Validate() tests. 38 | func TestValidate(t *testing.T) { 39 | t.Parallel() 40 | 41 | cases := []struct { 42 | Host *Host 43 | Message string 44 | Error bool 45 | }{ 46 | { 47 | &Host{ 48 | SSHConfig: &ssh.Config{}, 49 | DirectConfig: &direct.Config{}, 50 | }, 51 | "Validate should reject ambiguous configuration", 52 | true, 53 | }, 54 | { 55 | &Host{}, 56 | "Validate should reject empty configuration", 57 | true, 58 | }, 59 | { 60 | &Host{ 61 | SSHConfig: &ssh.Config{}, 62 | }, 63 | "Validate must validate ssh configuration", 64 | true, 65 | }, 66 | } 67 | 68 | for n, testCase := range cases { 69 | testCase := testCase 70 | 71 | t.Run(strconv.Itoa(n), func(t *testing.T) { 72 | t.Parallel() 73 | 74 | err := testCase.Host.Validate() 75 | if testCase.Error && err == nil { 76 | t.Fatal(testCase.Message) 77 | } 78 | 79 | if !testCase.Error && err != nil { 80 | t.Error(testCase.Message) 81 | } 82 | }) 83 | } 84 | } 85 | 86 | // Connect() tests. 87 | func TestConnect(t *testing.T) { 88 | t.Parallel() 89 | 90 | h := Host{ 91 | DirectConfig: &direct.Config{}, 92 | } 93 | 94 | c, err := h.New() 95 | if err != nil { 96 | t.Fatalf("Config should be valid, got: %v", err) 97 | } 98 | 99 | if _, err := c.Connect(); err != nil { 100 | t.Fatalf("Direct config should always connect, got: %v", err) 101 | } 102 | } 103 | 104 | // ForwardUnixSocket() tests. 105 | func TestForwardUnixSocket(t *testing.T) { 106 | t.Parallel() 107 | 108 | h := Host{ 109 | DirectConfig: &direct.Config{}, 110 | } 111 | 112 | c, err := h.New() 113 | if err != nil { 114 | t.Fatalf("Config should be valid, got: %v", err) 115 | } 116 | 117 | hc, err := c.Connect() 118 | if err != nil { 119 | t.Fatalf("Direct config should always connect, got: %v", err) 120 | } 121 | 122 | if _, err := hc.ForwardUnixSocket("unix:///nonexisting"); err != nil { 123 | t.Fatalf("Forwarding shouldn't fail, got: %v", err) 124 | } 125 | } 126 | 127 | // ForwardTCP() tests. 128 | func TestForwardTCP(t *testing.T) { 129 | t.Parallel() 130 | 131 | h := Host{ 132 | DirectConfig: &direct.Config{}, 133 | } 134 | 135 | c, err := h.New() 136 | if err != nil { 137 | t.Fatalf("Config should be valid, got: %v", err) 138 | } 139 | 140 | hc, err := c.Connect() 141 | if err != nil { 142 | t.Fatalf("Direct config should always connect, got: %v", err) 143 | } 144 | 145 | if _, err := hc.ForwardTCP("localhost:80"); err != nil { 146 | t.Fatalf("Forwarding shouldn't fail, got: %v", err) 147 | } 148 | } 149 | 150 | // BuildConfig() tests. 151 | func TestBuildConfigDirectByDefault(t *testing.T) { 152 | t.Parallel() 153 | 154 | h := BuildConfig(Host{}, Host{}) 155 | if err := h.Validate(); err != nil { 156 | t.Errorf("Config returned by default should be valid, got: %v", err) 157 | } 158 | 159 | if h.DirectConfig == nil { 160 | t.Fatalf("BuildConfig should return direct config by default") 161 | } 162 | } 163 | 164 | func TestBuildConfigFirstPriorityDirect(t *testing.T) { 165 | t.Parallel() 166 | 167 | firstHost := Host{ 168 | DirectConfig: &direct.Config{}, 169 | } 170 | 171 | secondHost := Host{ 172 | SSHConfig: &ssh.Config{}, 173 | } 174 | 175 | h := BuildConfig(firstHost, secondHost) 176 | if err := h.Validate(); err != nil { 177 | t.Errorf("Config returned by default should be valid, got: %v", err) 178 | } 179 | 180 | if h.SSHConfig != nil { 181 | t.Fatalf("BuildConfig should not inject SSH configuration from defaults if direct configuration has been requested") 182 | } 183 | } 184 | 185 | func TestBuildConfigFirstPriotitySSH(t *testing.T) { 186 | t.Parallel() 187 | 188 | firstHost := Host{ 189 | SSHConfig: &ssh.Config{ 190 | Address: "foo", 191 | Password: "foo", 192 | }, 193 | } 194 | 195 | secondHost := Host{ 196 | DirectConfig: &direct.Config{}, 197 | } 198 | 199 | h := BuildConfig(firstHost, secondHost) 200 | if err := h.Validate(); err != nil { 201 | t.Errorf("Config returned should be valid, got: %v", err) 202 | } 203 | 204 | if h.DirectConfig != nil { 205 | t.Fatalf("BuildConfig should not have direct configuration from defaults if SSH configuration has been requested") 206 | } 207 | } 208 | 209 | func TestBuildConfigSSH(t *testing.T) { 210 | t.Parallel() 211 | 212 | firstHost := Host{ 213 | SSHConfig: &ssh.Config{ 214 | Address: "foo", 215 | }, 216 | } 217 | 218 | secondHost := Host{ 219 | SSHConfig: &ssh.Config{ 220 | Port: 33, 221 | }, 222 | } 223 | 224 | if h := BuildConfig(firstHost, secondHost); h.SSHConfig.Port != 33 || h.SSHConfig.Address != "foo" { 225 | t.Fatalf("BuildConfig should merge ssh config, got: %+v", h) 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /internal/utiltest/x509.go: -------------------------------------------------------------------------------- 1 | // Package utiltest provides testing helpers, for generating valid mock data like 2 | // X509 certificates etc. 3 | package utiltest 4 | 5 | import ( 6 | "bytes" 7 | "crypto/ecdsa" 8 | "crypto/elliptic" 9 | "crypto/rand" 10 | "crypto/rsa" 11 | "crypto/x509" 12 | "crypto/x509/pkix" 13 | "encoding/pem" 14 | "fmt" 15 | "math/big" 16 | "testing" 17 | "time" 18 | ) 19 | 20 | const ( 21 | // CertValidityDuration is how long certificate is valid from the moment of generation. 22 | certValidityDuration = 1 * time.Hour 23 | ) 24 | 25 | // PKI struct holds X509 certificate and belonging RSA private key in PEM format. 26 | type PKI struct { 27 | Certificate string 28 | PrivateKey string 29 | } 30 | 31 | // GenerateX509Certificate generates random X.509 certificate and 32 | // returns it as string in PEM format. 33 | func GenerateX509Certificate(t *testing.T) string { 34 | t.Helper() 35 | 36 | return GeneratePKI(t).Certificate 37 | } 38 | 39 | // GenerateRSAPrivateKey generates RSA private key and returns it 40 | // as string in PEM format. 41 | func GenerateRSAPrivateKey(t *testing.T) string { 42 | t.Helper() 43 | 44 | return GeneratePKI(t).PrivateKey 45 | } 46 | 47 | // GeneratePKCS1PrivateKey generates RSA private key in PKCS1 format, 48 | // PEM encoded. 49 | func GeneratePKCS1PrivateKey(t *testing.T) string { 50 | t.Helper() 51 | 52 | keyBits := 2048 53 | 54 | priv, err := rsa.GenerateKey(rand.Reader, keyBits) 55 | if err != nil { 56 | t.Fatalf("Failed to generate RSA key: %v", err) 57 | } 58 | 59 | privBytes := x509.MarshalPKCS1PrivateKey(priv) 60 | 61 | var key bytes.Buffer 62 | if err := pem.Encode(&key, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: privBytes}); err != nil { 63 | t.Fatalf("Failed to write data to key.pem: %s", err) 64 | } 65 | 66 | return key.String() 67 | } 68 | 69 | // GenerateECPrivateKey generates EC private key, PEM encoded. 70 | func GenerateECPrivateKey(t *testing.T) string { 71 | t.Helper() 72 | 73 | priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) 74 | if err != nil { 75 | t.Fatalf("Failed generating ECDSA key: %v", err) 76 | } 77 | 78 | privBytes, err := x509.MarshalECPrivateKey(priv) 79 | if err != nil { 80 | t.Fatalf("Failed serializing EC private key: %v", err) 81 | } 82 | 83 | var key bytes.Buffer 84 | if err := pem.Encode(&key, &pem.Block{Type: "EC PRIVATE KEY", Bytes: privBytes}); err != nil { 85 | t.Fatalf("Failed to write data to key.pem: %s", err) 86 | } 87 | 88 | return key.String() 89 | } 90 | 91 | // GeneratePKI generates PKI struct. 92 | func GeneratePKI(t *testing.T) *PKI { 93 | t.Helper() 94 | 95 | p, err := GeneratePKIErr() 96 | if err != nil { 97 | t.Fatalf("Failed generating fake PKI: %v", err) 98 | } 99 | 100 | return p 101 | } 102 | 103 | // generateX509Certificate generates X.509 certificate in DER format using given RSA private key. 104 | func generateX509Certificate(priv *rsa.PrivateKey) ([]byte, error) { 105 | var serialNumberLimitBase uint = 128 106 | 107 | // Generate serial number for X.509 certificate. 108 | // 109 | //nolint:gomnd // As in https://golang.org/src/crypto/tls/generate_cert.go. 110 | serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), serialNumberLimitBase) 111 | 112 | serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) 113 | if err != nil { 114 | return nil, fmt.Errorf("generating serial number: %w", err) 115 | } 116 | 117 | template := x509.Certificate{ 118 | SerialNumber: serialNumber, 119 | Subject: pkix.Name{ 120 | Organization: []string{"example"}, 121 | }, 122 | NotBefore: time.Now(), 123 | NotAfter: time.Now().Add(certValidityDuration), 124 | 125 | KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, 126 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, 127 | BasicConstraintsValid: true, 128 | } 129 | 130 | // Create X.509 certificate in DER format. 131 | return x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) 132 | } 133 | 134 | // encodePKI converts RSA private key and X.509 certificate in DER format into PKI struct. 135 | func encodePKI(priv *rsa.PrivateKey, pub []byte) (*PKI, error) { 136 | // Encode private certificate into PEM format. 137 | var cert bytes.Buffer 138 | if err := pem.Encode(&cert, &pem.Block{Type: "CERTIFICATE", Bytes: pub}); err != nil { 139 | return nil, fmt.Errorf("writing data to cert.pem: %w", err) 140 | } 141 | 142 | // Convert RSA private key into PKCS8 DER format. 143 | privBytes, err := x509.MarshalPKCS8PrivateKey(priv) 144 | if err != nil { 145 | return nil, fmt.Errorf("marshaling private key: %w", err) 146 | } 147 | 148 | // Convert private key from PKCS8 DER format to PEM format. 149 | var key bytes.Buffer 150 | if err := pem.Encode(&key, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil { 151 | return nil, fmt.Errorf("writing data to key.pem: %w", err) 152 | } 153 | 154 | return &PKI{ 155 | Certificate: cert.String(), 156 | PrivateKey: key.String(), 157 | }, nil 158 | } 159 | 160 | // GeneratePKIErr generates fake PKI X.509 key pair sutiable for tests. 161 | func GeneratePKIErr() (*PKI, error) { 162 | keyBits := 2048 163 | 164 | // Generate RSA private key. 165 | priv, err := rsa.GenerateKey(rand.Reader, keyBits) 166 | if err != nil { 167 | return nil, fmt.Errorf("generating RSA key: %w", err) 168 | } 169 | 170 | derBytes, err := generateX509Certificate(priv) 171 | if err != nil { 172 | return nil, fmt.Errorf("creating X.509 certificate: %w", err) 173 | } 174 | 175 | return encodePKI(priv, derBytes) 176 | } 177 | -------------------------------------------------------------------------------- /pkg/container/runtime/docker/fake.go: -------------------------------------------------------------------------------- 1 | package docker 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "strings" 7 | 8 | dockertypes "github.com/docker/docker/api/types" 9 | containertypes "github.com/docker/docker/api/types/container" 10 | networktypes "github.com/docker/docker/api/types/network" 11 | v1 "github.com/opencontainers/image-spec/specs-go/v1" 12 | ) 13 | 14 | // FakeClient is a mock of Docker client, which should be used only for testing. 15 | type FakeClient struct { 16 | // ContainerCreateF will be called by ContainerCreate. 17 | ContainerCreateF func( 18 | ctx context.Context, 19 | config *containertypes.Config, 20 | hostConfig *containertypes.HostConfig, 21 | networkingConfig *networktypes.NetworkingConfig, 22 | platform *v1.Platform, 23 | containerName string, 24 | ) (containertypes.CreateResponse, error) 25 | 26 | // ContainerStartF will be called by ContainerStart. 27 | ContainerStartF func(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error 28 | 29 | // ContainerStopF will be called by ContainerStop. 30 | ContainerStopF func(ctx context.Context, container string, timeout containertypes.StopOptions) error 31 | 32 | // ContainerInspectF will be called by ContainerInspect. 33 | ContainerInspectF func(ctx context.Context, container string) (dockertypes.ContainerJSON, error) 34 | 35 | // ContainerRemoveF will be called by ContainerRemove. 36 | ContainerRemoveF func(ctx context.Context, container string, options dockertypes.ContainerRemoveOptions) error 37 | 38 | // CopyFromContainerF will be called by CopyFromContainer. 39 | CopyFromContainerF func( 40 | ctx context.Context, 41 | container string, 42 | srcPath string, 43 | ) (io.ReadCloser, dockertypes.ContainerPathStat, error) 44 | 45 | // CopyToContainerF will be called by CopyToContainer. 46 | CopyToContainerF func( 47 | ctx context.Context, 48 | container, 49 | path string, 50 | content io.Reader, 51 | options dockertypes.CopyToContainerOptions, 52 | ) error 53 | 54 | // ContainerStatPathF will be called by ContainerStatPath. 55 | ContainerStatPathF func(ctx context.Context, container, path string) (dockertypes.ContainerPathStat, error) 56 | 57 | // ImageListF will be called by ImageList. 58 | ImageListF func(ctx context.Context, options dockertypes.ImageListOptions) ([]dockertypes.ImageSummary, error) 59 | 60 | // ImagePullF will be called by ImagePull. 61 | ImagePullF func(ctx context.Context, ref string, options dockertypes.ImagePullOptions) (io.ReadCloser, error) 62 | } 63 | 64 | // ContainerCreate mocks Docker client ContainerCreate(). 65 | func (f *FakeClient) ContainerCreate( 66 | ctx context.Context, 67 | config *containertypes.Config, 68 | hostConfig *containertypes.HostConfig, 69 | networkingConfig *networktypes.NetworkingConfig, 70 | platform *v1.Platform, 71 | containerName string, 72 | ) (containertypes.CreateResponse, error) { 73 | return f.ContainerCreateF(ctx, config, hostConfig, networkingConfig, platform, containerName) 74 | } 75 | 76 | // ContainerStart mocks Docker client ContainerStart(). 77 | func (f *FakeClient) ContainerStart( 78 | ctx context.Context, 79 | container string, 80 | options dockertypes.ContainerStartOptions, 81 | ) error { 82 | return f.ContainerStartF(ctx, container, options) 83 | } 84 | 85 | // ContainerStop mocks Docker client ContainerStop(). 86 | func (f *FakeClient) ContainerStop(ctx context.Context, container string, options containertypes.StopOptions) error { 87 | return f.ContainerStopF(ctx, container, options) 88 | } 89 | 90 | // ContainerInspect mocks Docker client ContainerInspect(). 91 | func (f *FakeClient) ContainerInspect(ctx context.Context, container string) (dockertypes.ContainerJSON, error) { 92 | return f.ContainerInspectF(ctx, container) 93 | } 94 | 95 | // ContainerRemove mocks Docker client ContainerRemove(). 96 | func (f *FakeClient) ContainerRemove( 97 | ctx context.Context, 98 | container string, 99 | options dockertypes.ContainerRemoveOptions, 100 | ) error { 101 | return f.ContainerRemoveF(ctx, container, options) 102 | } 103 | 104 | // CopyFromContainer mocks Docker client CopyFromContainer(). 105 | func (f *FakeClient) CopyFromContainer( 106 | ctx context.Context, 107 | container, 108 | srcPath string, 109 | ) (io.ReadCloser, dockertypes.ContainerPathStat, error) { 110 | return f.CopyFromContainerF(ctx, container, srcPath) 111 | } 112 | 113 | // CopyToContainer mocks Docker client CopyToContainer(). 114 | func (f *FakeClient) CopyToContainer( 115 | ctx context.Context, 116 | container, 117 | path string, 118 | content io.Reader, 119 | options dockertypes.CopyToContainerOptions, 120 | ) error { 121 | return f.CopyToContainerF(ctx, container, path, content, options) 122 | } 123 | 124 | // ContainerStatPath mocks Docker client ContainerStatPath(). 125 | func (f *FakeClient) ContainerStatPath( 126 | ctx context.Context, 127 | container, 128 | path string, 129 | ) (dockertypes.ContainerPathStat, error) { 130 | return f.ContainerStatPathF(ctx, container, path) 131 | } 132 | 133 | // ImageList mocks Docker client ImageList(). 134 | func (f *FakeClient) ImageList( 135 | ctx context.Context, 136 | options dockertypes.ImageListOptions, 137 | ) ([]dockertypes.ImageSummary, error) { 138 | if f.ImageListF == nil { 139 | return []dockertypes.ImageSummary{}, nil 140 | } 141 | 142 | return f.ImageListF(ctx, options) 143 | } 144 | 145 | // ImagePull mocks Docker client ImagePull(). 146 | func (f *FakeClient) ImagePull( 147 | ctx context.Context, 148 | ref string, 149 | options dockertypes.ImagePullOptions, 150 | ) (io.ReadCloser, error) { 151 | if f.ImagePullF == nil { 152 | return io.NopCloser(strings.NewReader("")), nil 153 | } 154 | 155 | return f.ImagePullF(ctx, ref, options) 156 | } 157 | -------------------------------------------------------------------------------- /internal/util/util_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "reflect" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/logrusorgru/aurora" 9 | ) 10 | 11 | const ( 12 | expectedValueString = "foo" 13 | expectedValueInt = 10 14 | ) 15 | 16 | func TestPickStringLast(t *testing.T) { 17 | t.Parallel() 18 | 19 | if v := PickString("", "", expectedValueString); v != expectedValueString { 20 | t.Fatalf("Expected %s, got %s", expectedValueString, v) 21 | } 22 | } 23 | 24 | func TestPickStringNoValue(t *testing.T) { 25 | t.Parallel() 26 | 27 | if v := PickString(""); v != "" { 28 | t.Fatalf("Expected %q, got %q", "", v) 29 | } 30 | } 31 | 32 | func TestPickStringFirst(t *testing.T) { 33 | t.Parallel() 34 | 35 | if v := PickString(expectedValueString, "bar"); v != expectedValueString { 36 | t.Fatalf("Expected %s, got %s", expectedValueString, v) 37 | } 38 | } 39 | 40 | func TestPickIntLast(t *testing.T) { 41 | t.Parallel() 42 | 43 | if v := PickInt(0, 0, expectedValueInt); v != expectedValueInt { 44 | t.Fatalf("Expected %d, got %d", expectedValueInt, v) 45 | } 46 | } 47 | 48 | func TestPickIntNoValue(t *testing.T) { 49 | t.Parallel() 50 | 51 | if v := PickInt(0); v != 0 { 52 | t.Fatalf("Expected %d, got %d", 0, v) 53 | } 54 | } 55 | 56 | func TestPickIntFirst(t *testing.T) { 57 | t.Parallel() 58 | 59 | if v := PickInt(expectedValueInt, 5); v != expectedValueInt { 60 | t.Fatalf("Expected %d, got %d", expectedValueInt, v) 61 | } 62 | } 63 | 64 | func TestIndent(t *testing.T) { 65 | t.Parallel() 66 | 67 | if a, expected := Indent("foo", " "), " foo"; a != expected { 68 | t.Fatalf("Expected %q, got %q", expected, a) 69 | } 70 | } 71 | 72 | func TestIndentWithNewline(t *testing.T) { 73 | t.Parallel() 74 | 75 | expected := " foo\n bar\n" //nolint:ifshort // Declare 2 variables in if statement is not common. 76 | if a := Indent("foo\nbar\n", " "); a != expected { 77 | t.Fatalf("Expected %q, got %q", expected, a) 78 | } 79 | } 80 | 81 | func TestIndentEmpty(t *testing.T) { 82 | t.Parallel() 83 | 84 | expected := "" //nolint:ifshort // Declare 2 variables in if statement is not common. 85 | if a := Indent("", ""); a != expected { 86 | t.Fatalf("Expected %q, got %q", expected, a) 87 | } 88 | } 89 | 90 | func TestIndentEmptyText(t *testing.T) { 91 | t.Parallel() 92 | 93 | expected := "" //nolint:ifshort // Declare 2 variables in if statement is not common. 94 | if a := Indent("", " "); a != expected { 95 | t.Fatalf("Expected %q, got %q", expected, a) 96 | } 97 | } 98 | 99 | func TestIndentEmptyIndent(t *testing.T) { 100 | t.Parallel() 101 | 102 | expected := "foo\nbar" //nolint:ifshort // Declare 2 variables in if statement is not common. 103 | if a := Indent("foo\nbar", ""); a != expected { 104 | t.Fatalf("Expected %q, got %q", expected, a) 105 | } 106 | } 107 | 108 | func TestJoinSorted(t *testing.T) { 109 | t.Parallel() 110 | 111 | expected := "baz/doh|foo/bar" //nolint:ifshort // Declare 2 variables in if statement is not common. 112 | 113 | values := map[string]string{ 114 | "foo": "bar", 115 | "baz": "doh", 116 | } 117 | 118 | if a := JoinSorted(values, "/", "|"); a != expected { 119 | t.Fatalf("Expected %q, got %q", expected, a) 120 | } 121 | } 122 | 123 | func TestPickStringSlice(t *testing.T) { 124 | t.Parallel() 125 | 126 | expected := []string{"foo"} //nolint:ifshort // Declare 2 variables in if statement is not common. 127 | if v := PickStringSlice([]string{}, expected); !reflect.DeepEqual(v, expected) { 128 | t.Fatalf("Expected %v, got %v", expected, v) 129 | } 130 | } 131 | 132 | func TestPickStringMap(t *testing.T) { 133 | t.Parallel() 134 | 135 | expected := map[string]string{"foo": "bar"} 136 | if v := PickStringMap(map[string]string{}, expected); !reflect.DeepEqual(v, expected) { 137 | t.Fatalf("Expected %v, got %v", expected, v) 138 | } 139 | } 140 | 141 | func TestPickStringSliceEmpty(t *testing.T) { 142 | t.Parallel() 143 | 144 | expected := []string{} //nolint:ifshort // Declare 2 variables in if statement is not common. 145 | if v := PickStringSlice([]string{}, expected); !reflect.DeepEqual(v, expected) { 146 | t.Fatalf("Expected %v, got %v", expected, v) 147 | } 148 | } 149 | 150 | func TestPickStringMapEmpty(t *testing.T) { 151 | t.Parallel() 152 | 153 | expected := map[string]string{} 154 | if v := PickStringMap(map[string]string{}, expected); !reflect.DeepEqual(v, expected) { 155 | t.Fatalf("Expected %v, got %v", expected, v) 156 | } 157 | } 158 | 159 | func TestKeysStringMap(t *testing.T) { 160 | t.Parallel() 161 | 162 | expected := []string{"baz", "foo"} 163 | m := map[string]string{ 164 | "foo": "bar", 165 | "baz": "doh", 166 | } 167 | 168 | if k := KeysStringMap(m); !reflect.DeepEqual(expected, k) { 169 | t.Fatalf("Expected %v, got %v", expected, k) 170 | } 171 | } 172 | 173 | func TestColorizeDiff(t *testing.T) { 174 | t.Parallel() 175 | 176 | cases := []struct { 177 | input string 178 | output string 179 | }{ 180 | { 181 | input: "", 182 | output: "", 183 | }, 184 | { 185 | input: "\n", 186 | output: "\n", 187 | }, 188 | { 189 | input: "foo\n", 190 | output: "foo\n", 191 | }, 192 | { 193 | input: "+foo\n", 194 | output: aurora.Green("+foo\n").String(), 195 | }, 196 | { 197 | input: "-foo\n", 198 | output: aurora.Red("-foo\n").String(), 199 | }, 200 | { 201 | input: "foo\nbar", 202 | output: "foo\nbar", 203 | }, 204 | { 205 | input: "+foo\n-bar\nbaz\n", 206 | output: aurora.Green("+foo\n").String() + aurora.Red("-bar\n").String() + "baz\n", 207 | }, 208 | } 209 | 210 | for n, testCase := range cases { 211 | testCase := testCase 212 | 213 | t.Run(strconv.Itoa(n), func(t *testing.T) { 214 | t.Parallel() 215 | 216 | if result := ColorizeDiff(testCase.input); result != testCase.output { 217 | t.Errorf("Expected %q, got %q", testCase.output, result) 218 | } 219 | }) 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /pkg/kubernetes/client/client.go: -------------------------------------------------------------------------------- 1 | // Package client ships helper functions for building and using Kubernetes client. 2 | package client 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "strings" 9 | "time" 10 | 11 | v1 "k8s.io/api/core/v1" 12 | "k8s.io/apimachinery/pkg/api/errors" 13 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "k8s.io/apimachinery/pkg/types" 15 | "k8s.io/apimachinery/pkg/util/wait" 16 | "k8s.io/client-go/kubernetes" 17 | ) 18 | 19 | const ( 20 | // PollInterval defines how long we wait before next attempt while waiting for the objects. 21 | PollInterval = 5 * time.Second 22 | 23 | // RetryTimeout defines how long we wait before timing out waiting for the objects. 24 | RetryTimeout = 10 * time.Minute 25 | ) 26 | 27 | // Client defines exported capabilities of Flexkube k8s client. 28 | type Client interface { 29 | // CheckNodeExists returns a function, which checks, if given node exists. 30 | CheckNodeExists(name string) func() (bool, error) 31 | 32 | // CheckNodeReady returns a function, which checks, if given node is ready. 33 | CheckNodeReady(name string) func() (bool, error) 34 | 35 | // WaitForNode waits, until Node object shows up in the API. 36 | WaitForNode(name string) error 37 | 38 | // WaitForNodeReady waits, until Node object becomes ready. 39 | WaitForNodeReady(name string) error 40 | 41 | // LabelNode patches Node object to set given labels on it. 42 | LabelNode(name string, labels map[string]string) error 43 | 44 | // PingWait waits until API server becomes available. 45 | PingWait(pollInterval, retryTimeout time.Duration) error 46 | } 47 | 48 | type client struct { 49 | *kubernetes.Clientset 50 | } 51 | 52 | // NewClient takes content of kubeconfig file as an argument and returns flexkube kubernetes client, 53 | // which implements bunch of helper methods for Kubernetes API. 54 | func NewClient(kubeconfig []byte) (Client, error) { 55 | c, err := NewClientset(kubeconfig) 56 | if err != nil { 57 | return nil, fmt.Errorf("creating kubernetes clientset: %w", err) 58 | } 59 | 60 | return &client{c}, nil 61 | } 62 | 63 | // PingWait waits for Kubernetes API to become available. 64 | func (c *client) PingWait(pollInterval, retryTimeout time.Duration) error { 65 | //nolint:staticcheck // Will migrate once https://github.com/kubernetes/kubernetes/issues/119533 is resolved. 66 | return wait.PollImmediate(pollInterval, retryTimeout, c.ping) 67 | } 68 | 69 | // ping checks availability of Kubernetes API by fetching all Roles in kube-system namespace. 70 | // We use Roles, as helm client sometimes fails, even if API is already available, 71 | // saying that this type of object is not recognized. 72 | // 73 | //nolint:nilerr // Ignore errors here, as we only poke cluster to make sure it's functional. 74 | func (c *client) ping() (bool, error) { 75 | if _, err := c.RbacV1().Roles("").List(context.TODO(), metav1.ListOptions{}); err != nil { 76 | return false, nil 77 | } 78 | 79 | if _, err := c.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{}); err != nil { 80 | return false, nil 81 | } 82 | 83 | return true, nil 84 | } 85 | 86 | // CheckNodeExists checks if given node object exists. 87 | func (c *client) CheckNodeExists(name string) func() (bool, error) { 88 | return func() (bool, error) { 89 | _, err := c.CoreV1().Nodes().Get(context.TODO(), name, metav1.GetOptions{}) 90 | if err == nil { 91 | return true, nil 92 | } 93 | 94 | if errors.IsNotFound(err) { 95 | return false, nil 96 | } 97 | 98 | return false, fmt.Errorf("getting node %q: %w", name, err) 99 | } 100 | } 101 | 102 | // CheckNodeReady checks if given node object is ready. 103 | func (c *client) CheckNodeReady(name string) func() (bool, error) { 104 | return func() (bool, error) { 105 | n, err := c.CoreV1().Nodes().Get(context.TODO(), name, metav1.GetOptions{}) 106 | if err != nil { 107 | return false, nil //nolint:nilerr // Ignore all errors, node should eventually come up. 108 | } 109 | 110 | for _, condition := range n.Status.Conditions { 111 | if condition.Type == v1.NodeReady && condition.Status == v1.ConditionTrue { 112 | return true, nil 113 | } 114 | } 115 | 116 | return false, nil 117 | } 118 | } 119 | 120 | // WaitForNode waits for node object. If object is not found and we reach the timeout, error is returned. 121 | func (c *client) WaitForNode(name string) error { 122 | //nolint:staticcheck // Will migrate once https://github.com/kubernetes/kubernetes/issues/119533 is resolved. 123 | return wait.PollImmediate(PollInterval, RetryTimeout, c.CheckNodeExists(name)) 124 | } 125 | 126 | // WaitForNode waits for node object to become ready. If object is not found and we reach the timeout, 127 | // error is returned. 128 | func (c *client) WaitForNodeReady(name string) error { 129 | //nolint:staticcheck // Will migrate once https://github.com/kubernetes/kubernetes/issues/119533 is resolved. 130 | return wait.PollImmediate(PollInterval, RetryTimeout, c.CheckNodeReady(name)) 131 | } 132 | 133 | // LabelNode add specified labels to the Node object. If label already exist, it will be replaced. 134 | func (c *client) LabelNode(name string, labels map[string]string) error { 135 | if err := c.WaitForNode(name); err != nil { 136 | return fmt.Errorf("waiting for node: %w", err) 137 | } 138 | 139 | patches := []patchStringValue{} 140 | 141 | for k, v := range labels { 142 | patches = append(patches, patchStringValue{ 143 | Op: "replace", 144 | Path: fmt.Sprintf("/metadata/labels/%s", strings.ReplaceAll(strings.ReplaceAll(k, "~", "~0"), "/", "~1")), 145 | Value: v, 146 | }) 147 | } 148 | 149 | payloadBytes, err := json.Marshal(patches) 150 | if err != nil { 151 | return fmt.Errorf("encoding update payload: %w", err) 152 | } 153 | 154 | nc := c.CoreV1().Nodes() 155 | if _, err := nc.Patch(context.TODO(), name, types.JSONPatchType, payloadBytes, metav1.PatchOptions{}); err != nil { 156 | return fmt.Errorf("patching node %q: %w", name, err) 157 | } 158 | 159 | return nil 160 | } 161 | -------------------------------------------------------------------------------- /pkg/controlplane/controlplane_test.go: -------------------------------------------------------------------------------- 1 | package controlplane 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | "text/template" 8 | 9 | "github.com/flexkube/libflexkube/internal/util" 10 | "github.com/flexkube/libflexkube/internal/utiltest" 11 | "github.com/flexkube/libflexkube/pkg/pki" 12 | ) 13 | 14 | const controlplaneYAMLTemplate = ` 15 | common: 16 | kubernetesCACertificate: | 17 | {{.Certificate}} 18 | frontProxyCACertificate: | 19 | {{.Certificate}} 20 | kubeAPIServer: 21 | apiServerCertificate: | 22 | {{.Certificate}} 23 | apiServerKey: | 24 | {{.PrivateKey}} 25 | frontProxyCertificate: | 26 | {{.Certificate}} 27 | frontProxyKey: | 28 | {{.PrivateKey}} 29 | kubeletClientCertificate: | 30 | {{.Certificate}} 31 | kubeletClientKey: | 32 | {{.PrivateKey}} 33 | serviceAccountPrivateKey: foo 34 | serviceCIDR: 11.0.0.0/24 35 | etcdCACertificate: | 36 | {{.Certificate}} 37 | etcdClientCertificate: | 38 | {{.Certificate}} 39 | etcdClientKey: | 40 | {{.PrivateKey}} 41 | etcdServers: 42 | - http://10.0.2.15:2379 43 | bindAddress: 0.0.0.0 44 | advertiseAddress: 127.0.0.1 45 | kubeControllerManager: 46 | kubernetesCAKey: | 47 | {{.PrivateKey}} 48 | serviceAccountPrivateKey: | 49 | {{.PrivateKey}} 50 | kubeconfig: 51 | clientCertificate: | 52 | {{.CertificateDeep}} 53 | clientKey: | 54 | {{.PrivateKeyDeep}} 55 | rootCACertificate: | 56 | {{.Certificate}} 57 | apiServerAddress: 127.0.0.1 58 | kubeScheduler: 59 | kubeconfig: 60 | clientCertificate: | 61 | {{.CertificateDeep}} 62 | clientKey: | 63 | {{.PrivateKeyDeep}} 64 | apiServerPort: 6443 65 | ssh: 66 | user: "core" 67 | address: 127.0.0.1 68 | port: 2222 69 | password: "foo" 70 | connectionTimeout: 1ms 71 | retryTimeout: 1ms 72 | retryInterval: 1ms 73 | ` 74 | 75 | func controlplaneYAML(t *testing.T) string { 76 | t.Helper() 77 | 78 | pki := utiltest.GeneratePKI(t) 79 | 80 | data := struct { 81 | Certificate string 82 | PrivateKey string 83 | CertificateDeep string 84 | PrivateKeyDeep string 85 | }{ 86 | strings.TrimSpace(util.Indent(utiltest.GenerateX509Certificate(t), " ")), 87 | strings.TrimSpace(util.Indent(utiltest.GenerateRSAPrivateKey(t), " ")), 88 | strings.TrimSpace(util.Indent(pki.Certificate, " ")), 89 | strings.TrimSpace(util.Indent(pki.PrivateKey, " ")), 90 | } 91 | 92 | var buf bytes.Buffer 93 | 94 | tpl := template.Must(template.New("c").Parse(controlplaneYAMLTemplate)) 95 | if err := tpl.Execute(&buf, data); err != nil { 96 | t.Fatalf("Failed to generate config from template: %v", err) 97 | } 98 | 99 | return buf.String() 100 | } 101 | 102 | func TestControlplaneFromYaml(t *testing.T) { 103 | t.Parallel() 104 | 105 | testControlplane, err := FromYaml([]byte(controlplaneYAML(t))) 106 | if err != nil { 107 | t.Fatalf("Creating controlplane from YAML should succeed, got: %v", err) 108 | } 109 | 110 | if cc := testControlplane.Containers(); cc == nil { 111 | t.Fatalf("Containers() should return non-nil value") 112 | } 113 | 114 | if _, err := testControlplane.StateToYaml(); err != nil { 115 | t.Fatalf("Dumping state to YAML should work, got: %v", err) 116 | } 117 | 118 | if err := testControlplane.CheckCurrentState(); err != nil { 119 | t.Fatalf("Checking current state of empty controlplane should work, got: %v", err) 120 | } 121 | 122 | if err := testControlplane.Deploy(); err == nil { 123 | t.Fatalf("Deploying in testing environment should fail") 124 | } 125 | } 126 | 127 | // New() tests. 128 | func TestControlplaneNewValidate(t *testing.T) { 129 | t.Parallel() 130 | 131 | c := &Controlplane{} 132 | 133 | if _, err := c.New(); err == nil { 134 | t.Fatalf("New should validate controlplane configuration and fail on empty one") 135 | } 136 | } 137 | 138 | func TestControlplaneDestroyNoState(t *testing.T) { 139 | t.Parallel() 140 | 141 | testConfigRaw := controlplaneYAML(t) 142 | 143 | testConfigRaw += `destroy: true` 144 | 145 | if _, err := FromYaml([]byte(testConfigRaw)); err == nil { 146 | t.Fatalf("Creating controlplane config to destroy without state should fail") 147 | } 148 | } 149 | 150 | func TestControlplaneDestroyValidateState(t *testing.T) { 151 | t.Parallel() 152 | 153 | testConfigRaw := controlplaneYAML(t) 154 | 155 | testConfigRaw += `destroy: true 156 | state: 157 | foo: {} 158 | ` 159 | 160 | if _, err := FromYaml([]byte(testConfigRaw)); err == nil { 161 | t.Fatalf("Creating controlplane config to destroy with invalid state should fail") 162 | } 163 | } 164 | 165 | func TestControlplaneDestroyValidState(t *testing.T) { 166 | t.Parallel() 167 | 168 | testConfigRaw := `destroy: true 169 | state: 170 | foo: 171 | host: 172 | direct: {} 173 | container: 174 | runtime: 175 | docker: 176 | host: unix:///nonexistent 177 | config: 178 | name: foo 179 | image: busybox 180 | status: 181 | id: foo 182 | status: running 183 | ` 184 | 185 | if _, err := FromYaml([]byte(testConfigRaw)); err != nil { 186 | t.Fatalf("Creating controlplane config to destroy with only state should succeed, got: %v", err) 187 | } 188 | } 189 | 190 | func TestControlplaneNewPKIIntegration(t *testing.T) { 191 | t.Parallel() 192 | 193 | pki := &pki.PKI{ 194 | Etcd: &pki.Etcd{ 195 | ClientCNs: []string{"kube-apiserver", "root"}, 196 | }, 197 | Kubernetes: &pki.Kubernetes{}, 198 | } 199 | 200 | if err := pki.Generate(); err != nil { 201 | t.Fatalf("Generating PKI should succeed, got: %v", err) 202 | } 203 | 204 | testConfig := &Controlplane{ 205 | PKI: pki, 206 | APIServerAddress: "127.0.0.1", 207 | APIServerPort: 6443, 208 | KubeAPIServer: KubeAPIServer{ 209 | EtcdServers: []string{"https://127.0.0.1:2379"}, 210 | }, 211 | } 212 | 213 | if _, err := testConfig.New(); err != nil { 214 | t.Fatalf("Creating new controlplane with valid PKI should succeed, got: %v", err) 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /pkg/pki/etcd.go: -------------------------------------------------------------------------------- 1 | package pki 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | const ( 8 | // EtcdCACN is a default CN for etcd CA certificate, as recommended by 9 | // the https://kubernetes.io/docs/setup/best-practices/certificates/. 10 | EtcdCACN = "etcd-ca" 11 | ) 12 | 13 | // Etcd stores etcd PKI and their settings. 14 | type Etcd struct { 15 | // Inline Certificate struct, so some settings can be applied as defaults for all etcd certificates. 16 | Certificate 17 | 18 | // CA stores etcd CA certificate. 19 | CA *Certificate `json:"ca,omitempty"` 20 | 21 | // Peers is a map of peer certificates to generate, where key is name of the peer and value 22 | // is the IP address on which peer will be listening on. 23 | Peers map[string]string `json:"peers,omitempty"` 24 | 25 | // Servers is a map of server certificates to generate, where key is the CN of the client 26 | // certificate and value is the IP address on which the server will be listening on. 27 | Servers map[string]string `json:"servers,omitempty"` 28 | 29 | // ClientCNS is a list of client certificate Common Names to generate. 30 | ClientCNs []string `json:"clientCNs,omitempty"` 31 | 32 | // PeerCertificates defines and stores all peer certificates. 33 | PeerCertificates map[string]*Certificate `json:"peerCertificates,omitempty"` 34 | 35 | // ServerCertificates defines and stores all server certificates. 36 | ServerCertificates map[string]*Certificate `json:"serverCertificates,omitempty"` 37 | 38 | // ClientCertificates defined and stores all client certificates. 39 | ClientCertificates map[string]*Certificate `json:"clientCertificates,omitempty"` 40 | } 41 | 42 | // Generate generates etcd PKI. 43 | func (e *Etcd) Generate(rootCA *Certificate, defaultCertificate Certificate) error { 44 | if e.CA == nil { 45 | e.CA = &Certificate{} 46 | } 47 | 48 | servers := e.Servers 49 | 50 | // If there is no different server certificates defined, assume they are the same as peers. 51 | if e.Servers == nil && e.Peers != nil { 52 | servers = e.Peers 53 | } 54 | 55 | e.initializeCertificatesMaps(servers) 56 | 57 | caRequest := &certificateRequest{ 58 | Target: e.CA, 59 | CA: rootCA, 60 | Certificates: []*Certificate{ 61 | &defaultCertificate, 62 | &e.Certificate, 63 | caCertificate(EtcdCACN), 64 | e.CA, 65 | }, 66 | } 67 | 68 | // etcd CA Certificate 69 | if err := buildAndGenerate(caRequest); err != nil { 70 | return fmt.Errorf("generating etcd CA certificate: %w", err) 71 | } 72 | 73 | crs := []*certificateRequest{} 74 | 75 | crs = append(crs, e.crsFromMap(&defaultCertificate, e.PeerCertificates, e.Peers, true)...) 76 | crs = append(crs, e.crsFromMap(&defaultCertificate, e.ServerCertificates, servers, true)...) 77 | 78 | clientCNsMap := map[string]string{} 79 | for _, commonName := range e.ClientCNs { 80 | clientCNsMap[commonName] = "" 81 | } 82 | 83 | crs = append(crs, e.crsFromMap(&defaultCertificate, e.ClientCertificates, clientCNsMap, false)...) 84 | 85 | return buildAndGenerate(crs...) 86 | } 87 | 88 | func (e *Etcd) initializeCertificatesMaps(servers map[string]string) { 89 | if e.PeerCertificates == nil && len(e.Peers) != 0 { 90 | e.PeerCertificates = map[string]*Certificate{} 91 | } 92 | 93 | if e.ServerCertificates == nil && len(servers) != 0 { 94 | e.ServerCertificates = map[string]*Certificate{} 95 | } 96 | 97 | if e.ClientCertificates == nil && len(e.ClientCNs) != 0 { 98 | e.ClientCertificates = map[string]*Certificate{} 99 | } 100 | } 101 | 102 | // certificateFromCNIPMap produces a certificate from given common name and IP address. 103 | func certificateFromCNIPMap(commonName, ipAddress string, server bool) *Certificate { 104 | cert := &Certificate{ 105 | CommonName: commonName, 106 | KeyUsage: clientUsage(), 107 | } 108 | 109 | if server { 110 | cert.KeyUsage = clientServerUsage() 111 | cert.DNSNames = []string{commonName, "localhost"} 112 | } 113 | 114 | if ipAddress != "" && server { 115 | cert.IPAddresses = append(cert.IPAddresses, ipAddress, "127.0.0.1") 116 | } 117 | 118 | return cert 119 | } 120 | 121 | // peerCRs builds list of certificate requests for peer certificates by combining 122 | // information from PeerCertificates and Peers fields, where PeerCertificates always 123 | // takes precedence. 124 | func (e *Etcd) crsFromMap( 125 | defaultCertificate *Certificate, 126 | certs map[string]*Certificate, 127 | cnIPs map[string]string, 128 | server bool, 129 | ) []*certificateRequest { 130 | // Store peer CRs in temporary map, so we can find them by common name. 131 | crs := map[string]*certificateRequest{} 132 | 133 | // Iterate over peer certificates, as they should take priority over 134 | // Peers field. 135 | for commonName := range certs { 136 | crs[commonName] = &certificateRequest{ 137 | Target: certs[commonName], 138 | CA: e.CA, 139 | Certificates: []*Certificate{ 140 | defaultCertificate, 141 | &e.Certificate, 142 | certificateFromCNIPMap(commonName, cnIPs[commonName], server), 143 | certs[commonName], 144 | }, 145 | } 146 | } 147 | 148 | for commonName, ipAddress := range cnIPs { 149 | // If certificate request is already created for a given common name, it will 150 | // have peers information included, so we jump to another one. 151 | if _, ok := crs[commonName]; ok { 152 | continue 153 | } 154 | 155 | // Make sure target certificate is initialized. 156 | if _, ok := certs[commonName]; !ok { 157 | certs[commonName] = &Certificate{} 158 | } 159 | 160 | crs[commonName] = &certificateRequest{ 161 | Target: certs[commonName], 162 | CA: e.CA, 163 | Certificates: []*Certificate{ 164 | defaultCertificate, 165 | &e.Certificate, 166 | certificateFromCNIPMap(commonName, ipAddress, server), 167 | }, 168 | } 169 | } 170 | 171 | return certificateRequestsFromMap(crs) 172 | } 173 | 174 | func certificateRequestsFromMap(crsMap map[string]*certificateRequest) []*certificateRequest { 175 | crs := []*certificateRequest{} 176 | 177 | for _, cr := range crsMap { 178 | crs = append(crs, cr) 179 | } 180 | 181 | return crs 182 | } 183 | -------------------------------------------------------------------------------- /pkg/controlplane/kube-controller-manager_test.go: -------------------------------------------------------------------------------- 1 | package controlplane 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flexkube/libflexkube/internal/utiltest" 7 | "github.com/flexkube/libflexkube/pkg/host" 8 | "github.com/flexkube/libflexkube/pkg/host/transport/direct" 9 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 10 | "github.com/flexkube/libflexkube/pkg/types" 11 | ) 12 | 13 | //nolint:funlen // Just a lot of test cases. 14 | func TestKubeControllerManagerValidate(t *testing.T) { 15 | t.Parallel() 16 | 17 | hostConfig := &host.Host{ 18 | DirectConfig: &direct.Config{}, 19 | } 20 | 21 | pki := utiltest.GeneratePKI(t) 22 | 23 | kubeconfig := client.Config{ 24 | Server: "localhost", 25 | CACertificate: types.Certificate(pki.Certificate), 26 | ClientCertificate: types.Certificate(pki.Certificate), 27 | ClientKey: types.PrivateKey(pki.PrivateKey), 28 | } 29 | 30 | common := &Common{ 31 | KubernetesCACertificate: types.Certificate(pki.Certificate), 32 | FrontProxyCACertificate: types.Certificate(pki.Certificate), 33 | } 34 | 35 | cases := map[string]struct { 36 | Config *KubeControllerManager 37 | Error bool 38 | }{ 39 | "require Kubeconfig": { 40 | Config: &KubeControllerManager{ 41 | KubernetesCAKey: types.PrivateKey(pki.PrivateKey), 42 | ServiceAccountPrivateKey: types.PrivateKey(pki.PrivateKey), 43 | RootCACertificate: types.Certificate(pki.Certificate), 44 | Host: hostConfig, 45 | Common: common, 46 | }, 47 | Error: true, 48 | }, 49 | "require KubernetesCAKey": { 50 | Config: &KubeControllerManager{ 51 | ServiceAccountPrivateKey: types.PrivateKey(pki.PrivateKey), 52 | RootCACertificate: types.Certificate(pki.Certificate), 53 | Host: hostConfig, 54 | Kubeconfig: kubeconfig, 55 | Common: common, 56 | }, 57 | Error: true, 58 | }, 59 | "require ServiceAccountPrivateKey": { 60 | Config: &KubeControllerManager{ 61 | KubernetesCAKey: types.PrivateKey(pki.PrivateKey), 62 | RootCACertificate: types.Certificate(pki.Certificate), 63 | Host: hostConfig, 64 | Kubeconfig: kubeconfig, 65 | Common: common, 66 | }, 67 | Error: true, 68 | }, 69 | "require RootCACertificate": { 70 | Config: &KubeControllerManager{ 71 | KubernetesCAKey: types.PrivateKey(pki.PrivateKey), 72 | ServiceAccountPrivateKey: types.PrivateKey(pki.PrivateKey), 73 | Host: hostConfig, 74 | Kubeconfig: kubeconfig, 75 | Common: common, 76 | }, 77 | Error: true, 78 | }, 79 | "no host": { 80 | Config: &KubeControllerManager{ 81 | KubernetesCAKey: types.PrivateKey(pki.PrivateKey), 82 | ServiceAccountPrivateKey: types.PrivateKey(pki.PrivateKey), 83 | RootCACertificate: types.Certificate(pki.Certificate), 84 | Kubeconfig: kubeconfig, 85 | Common: common, 86 | }, 87 | Error: true, 88 | }, 89 | "bad host": { 90 | Config: &KubeControllerManager{ 91 | KubernetesCAKey: types.PrivateKey(pki.PrivateKey), 92 | ServiceAccountPrivateKey: types.PrivateKey(pki.PrivateKey), 93 | RootCACertificate: types.Certificate(pki.Certificate), 94 | Kubeconfig: kubeconfig, 95 | Host: &host.Host{}, 96 | }, 97 | Error: true, 98 | }, 99 | "valid": { 100 | Config: &KubeControllerManager{ 101 | KubernetesCAKey: types.PrivateKey(pki.PrivateKey), 102 | ServiceAccountPrivateKey: types.PrivateKey(pki.PrivateKey), 103 | RootCACertificate: types.Certificate(pki.Certificate), 104 | Host: hostConfig, 105 | Kubeconfig: kubeconfig, 106 | Common: common, 107 | }, 108 | Error: false, 109 | }, 110 | } 111 | 112 | for n, testCase := range cases { 113 | testCase := testCase 114 | 115 | t.Run(n, func(t *testing.T) { 116 | t.Parallel() 117 | 118 | err := testCase.Config.Validate() 119 | if !testCase.Error && err != nil { 120 | t.Errorf("Didn't expect error, got: %v", err) 121 | } 122 | 123 | if testCase.Error && err == nil { 124 | t.Errorf("Expected error") 125 | } 126 | }) 127 | } 128 | } 129 | 130 | func TestKubeControllerManagerToHostConfiguredContainer(t *testing.T) { 131 | t.Parallel() 132 | 133 | pki := utiltest.GeneratePKI(t) 134 | 135 | kcm := &KubeControllerManager{ 136 | KubernetesCAKey: types.PrivateKey(pki.PrivateKey), 137 | ServiceAccountPrivateKey: types.PrivateKey(pki.PrivateKey), 138 | RootCACertificate: types.Certificate(pki.Certificate), 139 | Host: &host.Host{ 140 | DirectConfig: &direct.Config{}, 141 | }, 142 | Kubeconfig: client.Config{ 143 | Server: "localhost", 144 | CACertificate: types.Certificate(pki.Certificate), 145 | ClientCertificate: types.Certificate(pki.Certificate), 146 | ClientKey: types.PrivateKey(pki.PrivateKey), 147 | }, 148 | } 149 | 150 | o, err := kcm.New() 151 | if err != nil { 152 | t.Fatalf("New should not return error, got: %v", err) 153 | } 154 | 155 | hcc, err := o.ToHostConfiguredContainer() 156 | if err != nil { 157 | t.Fatalf("Generating HostConfiguredContainer should work, got: %v", err) 158 | } 159 | 160 | if _, err := hcc.New(); err != nil { 161 | t.Fatalf("ToHostConfiguredContainer() should generate valid HostConfiguredContainer, got: %v", err) 162 | } 163 | 164 | if hcc.Container.Config.Image == "" { 165 | t.Fatalf("New() should set default image if it's not present") 166 | } 167 | } 168 | 169 | // New() tests. 170 | func TestKubeControllerManagerNewEmptyHost(t *testing.T) { 171 | t.Parallel() 172 | 173 | ks := &KubeControllerManager{} 174 | 175 | k, err := ks.New() 176 | if err == nil { 177 | t.Errorf("Attempting to create kube-scheduler from empty config should fail") 178 | } 179 | 180 | if k != nil { 181 | t.Fatalf("Failed attempt of creating kube-scheduler should not return kube-scheduler object") 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - master 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | container: golangci/golangci-lint:v1.61.0 11 | steps: 12 | - uses: actions/cache@v2 13 | with: 14 | path: | 15 | ~/go/pkg/mod 16 | ~/.cache/go-build 17 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 18 | restore-keys: | 19 | ${{ runner.os }}-go- 20 | - uses: actions/checkout@v2 21 | - run: make install-changelog BIN_PATH=/usr/local/bin 22 | - run: | 23 | # Required to avoid error 'fatal: detected dubious ownership in repository at' while calling 24 | # 'git status --porcelain'. 25 | git config --global --add safe.directory /__w/libflexkube/libflexkube 26 | make build build-test test-update-linters lint test-tidy test-changelog 27 | test: 28 | runs-on: ubuntu-latest 29 | steps: 30 | - uses: actions/setup-go@v2 31 | with: 32 | go-version: '1.23' 33 | - uses: actions/cache@v2 34 | with: 35 | path: | 36 | ~/go/pkg/mod 37 | ~/.cache/go-build 38 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 39 | restore-keys: | 40 | ${{ runner.os }}-go- 41 | - uses: actions/checkout@v2 42 | with: 43 | # Required for Codecov report uploading. 44 | fetch-depth: 0 45 | - name: Install required binaries 46 | run: | 47 | docker compose -f e2e/docker-compose.yml up generate-secrets 48 | sudo chown $(whoami): ./e2e/secrets/* 49 | docker compose -f e2e/docker-compose.yml up -d 50 | make install-cc-test-reporter BIN_PATH=/usr/local/bin 51 | - run: | 52 | export TEST_INTEGRATION_SSH_PORT=2222 53 | export TEST_INTEGRATION_SSH_PASSWORD_FILE=$(pwd)/e2e/secrets/password 54 | export TEST_INTEGRATION_SSH_PRIVATE_KEY_PATH=$(pwd)/e2e/secrets/id_rsa 55 | make test-integration-cover-upload 56 | build-docker: 57 | runs-on: ubuntu-latest 58 | steps: 59 | - uses: actions/checkout@v2 60 | - run: make build-docker 61 | build-integration-image: 62 | runs-on: ubuntu-latest 63 | steps: 64 | - uses: actions/checkout@v2 65 | - run: make build-integration 66 | build-e2e-image: 67 | runs-on: ubuntu-latest 68 | steps: 69 | - uses: actions/checkout@v2 70 | - run: make build-e2e 71 | e2e: 72 | runs-on: ubuntu-latest 73 | steps: 74 | - uses: actions/setup-go@v2 75 | with: 76 | go-version: '1.23' 77 | - uses: actions/cache@v2 78 | with: 79 | path: | 80 | ~/go/pkg/mod 81 | ~/.cache/go-build 82 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 83 | restore-keys: | 84 | ${{ runner.os }}-go- 85 | - uses: actions/checkout@v2 86 | - name: Setup containerd 87 | run: | 88 | # From https://docs.docker.com/engine/install/ubuntu/. 89 | sudo apt-get remove docker docker-engine docker.io containerd runc 90 | sudo apt-get update 91 | sudo apt-get install \ 92 | ca-certificates \ 93 | curl \ 94 | gnupg \ 95 | lsb-release 96 | sudo mkdir -p /etc/apt/keyrings 97 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg 98 | echo \ 99 | "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ 100 | $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null 101 | sudo apt-get update 102 | sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin 103 | # From Flatcar default configuration and image-builder containerd role. 104 | sudo sed -i 's/^disabled_plugins.*/disabled_plugins = []/g' /etc/containerd/config.toml 105 | cat < e2e/test-config.yaml 117 | sshPrivateKeyPath: $(pwd)/e2e/secrets/id_rsa 118 | nodesCIDR: 172.17.0.0/24 119 | nodeSSHPort: 2222 120 | workersCount: 0 121 | cidrIPsOffset: 1 122 | kubeletExtraArgs: 123 | - --fail-swap-on=false 124 | - --container-runtime-endpoint=unix:///run/containerd/containerd.sock 125 | cgroupDriver: cgroupfs 126 | EOF 127 | helm repo add flexkube https://flexkube.github.io/charts/ 128 | sudo chown $(whoami): e2e/secrets/id_rsa 129 | - run: make test-e2e 130 | codespell: 131 | runs-on: ubuntu-latest 132 | steps: 133 | - uses: actions/checkout@v2 134 | - uses: codespell-project/actions-codespell@master 135 | with: 136 | skip: .git,go.sum 137 | ignore_words_list: uptodate,decorder,complies 138 | check_filenames: true 139 | check_hidden: true 140 | vagrant: 141 | runs-on: ubuntu-latest 142 | steps: 143 | - uses: actions/checkout@v2 144 | - name: Install vagrant 145 | run: | 146 | curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - 147 | sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" 148 | sudo apt-get update && sudo apt-get install vagrant 149 | - run: make test-vagrant 150 | terraform: 151 | runs-on: ubuntu-latest 152 | steps: 153 | - uses: actions/checkout@v2 154 | - name: Get dependencies 155 | run: | 156 | sudo apt-get update && sudo apt-get install -y gnupg software-properties-common curl 157 | curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - 158 | sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" 159 | sudo apt-get update && sudo apt-get install terraform 160 | - run: | 161 | terraform -chdir=libvirt init 162 | make test-terraform 163 | semgrep: 164 | runs-on: ubuntu-latest 165 | container: 166 | image: returntocorp/semgrep 167 | steps: 168 | - uses: actions/checkout@v2 169 | - run: semgrep ci --config .semgrep.yml 170 | -------------------------------------------------------------------------------- /pkg/kubelet/pool_test.go: -------------------------------------------------------------------------------- 1 | package kubelet_test 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | "text/template" 8 | 9 | "github.com/flexkube/libflexkube/internal/util" 10 | "github.com/flexkube/libflexkube/internal/utiltest" 11 | "github.com/flexkube/libflexkube/pkg/kubelet" 12 | "github.com/flexkube/libflexkube/pkg/kubernetes/client" 13 | "github.com/flexkube/libflexkube/pkg/pki" 14 | "github.com/flexkube/libflexkube/pkg/types" 15 | ) 16 | 17 | func getPool(t *testing.T) types.Resource { 18 | t.Helper() 19 | 20 | configTemplate := ` 21 | ssh: 22 | address: localhost 23 | password: foo 24 | connectionTimeout: 1s 25 | retryTimeout: 1s 26 | retryInterval: 1s 27 | bootstrapConfig: 28 | server: "foo" 29 | token: "foo" 30 | volumePluginDir: /var/lib/kubelet/volumeplugins 31 | extraMounts: 32 | - source: /foo/ 33 | target: /bar 34 | kubernetesCACertificate: | 35 | {{.}} 36 | waitForNodeReady: false 37 | extraArgs: 38 | - --baz 39 | kubelets: 40 | - name: foo 41 | - name: bar 42 | extraMounts: 43 | - source: /doh/ 44 | target: /tmp 45 | extraArgs: 46 | - --bar 47 | ` 48 | 49 | var buf bytes.Buffer 50 | 51 | tpl := template.Must(template.New("c").Parse(configTemplate)) 52 | if err := tpl.Execute(&buf, strings.TrimSpace(util.Indent(utiltest.GenerateX509Certificate(t), " "))); err != nil { 53 | t.Fatalf("Failed to generate config from template: %v", err) 54 | } 55 | 56 | p, err := kubelet.FromYaml(buf.Bytes()) 57 | if err != nil { 58 | t.Fatalf("Creating pool from YAML should succeed, got: %v", err) 59 | } 60 | 61 | return p 62 | } 63 | 64 | // New() tests. 65 | func TestPoolNewValidate(t *testing.T) { 66 | t.Parallel() 67 | 68 | testConfigRaw := ` 69 | ssh: 70 | address: localhost 71 | password: foo 72 | connectionTimeout: 1s 73 | retryTimeout: 1s 74 | retryInterval: 1s 75 | volumePluginDir: /var/lib/kubelet/volumeplugins 76 | kubelets: 77 | - name: foo 78 | ` 79 | 80 | if _, err := kubelet.FromYaml([]byte(testConfigRaw)); err == nil { 81 | t.Fatalf("Creating pool from bad YAML should fail") 82 | } 83 | } 84 | 85 | // FromYaml() tests. 86 | func TestPoolFromYaml(t *testing.T) { 87 | t.Parallel() 88 | 89 | getPool(t) 90 | } 91 | 92 | // StateToYaml() tests. 93 | func TestPoolStateToYAML(t *testing.T) { 94 | t.Parallel() 95 | 96 | p := getPool(t) 97 | 98 | if _, err := p.StateToYaml(); err != nil { 99 | t.Fatalf("Dumping state to YAML should work, got: %v", err) 100 | } 101 | } 102 | 103 | // CheckCurrentState() tests. 104 | func TestPoolCheckCurrentState(t *testing.T) { 105 | t.Parallel() 106 | 107 | p := getPool(t) 108 | 109 | if err := p.CheckCurrentState(); err != nil { 110 | t.Fatalf("Checking current state of empty pool should work, got: %v", err) 111 | } 112 | } 113 | 114 | // Containers() tests. 115 | func TestPoolContainers(t *testing.T) { 116 | t.Parallel() 117 | 118 | p := getPool(t) 119 | 120 | if c := p.Containers(); c == nil { 121 | t.Fatalf("Containers() should return non-nil value") 122 | } 123 | } 124 | 125 | // Deploy() tests. 126 | func TestPoolDeploy(t *testing.T) { 127 | t.Parallel() 128 | 129 | p := getPool(t) 130 | 131 | if err := p.Deploy(); err == nil { 132 | t.Fatalf("Deploying in testing environment should fail") 133 | } 134 | } 135 | 136 | func Test_Pool_propagates_extra_mounts_to_members_without_extra_mounts_defined(t *testing.T) { 137 | t.Parallel() 138 | 139 | p := getPool(t) 140 | 141 | found := false 142 | 143 | for _, v := range p.Containers().DesiredState()["0"].Container.Config.Mounts { 144 | if v.Source == "/foo/" && v.Target == "/bar" { 145 | found = true 146 | } 147 | } 148 | 149 | if !found { 150 | t.Fatal("Kubelet foo should have propagated extra mount") 151 | } 152 | } 153 | 154 | func Test_Pool_retains_individual_members_extra_mounts(t *testing.T) { 155 | t.Parallel() 156 | 157 | p := getPool(t) 158 | 159 | found := false 160 | 161 | for _, v := range p.Containers().DesiredState()["1"].Container.Config.Mounts { 162 | if v.Source == "/doh/" && v.Target == "/tmp" { 163 | found = true 164 | } 165 | 166 | if v.Source == "/foo/" && v.Target == "/bar" { 167 | t.Errorf("Kubelet doh should not have propagated mounts") 168 | } 169 | } 170 | 171 | if !found { 172 | t.Fatalf("Kubelet doh should have directly configured extra mount") 173 | } 174 | } 175 | 176 | func Test_Pool_does_propagate_extra_args_when_instance_has_no_extra_args_set(t *testing.T) { 177 | t.Parallel() 178 | 179 | p := getPool(t) 180 | 181 | found := false 182 | 183 | for _, v := range p.Containers().DesiredState()["0"].Container.Config.Args { 184 | if v == "--baz" { 185 | found = true 186 | } 187 | } 188 | 189 | if !found { 190 | t.Errorf("Kubelet foo should have propagated extra arguments") 191 | } 192 | } 193 | 194 | func Test_Pool_does_preserve_extra_args_defined_in_instance(t *testing.T) { 195 | t.Parallel() 196 | 197 | p := getPool(t) 198 | 199 | found := false 200 | 201 | for _, arg := range p.Containers().DesiredState()["1"].Container.Config.Args { 202 | if arg == "--bar" { 203 | found = true 204 | } 205 | 206 | if arg == "--baz" { 207 | t.Errorf("Kubelet doh should not have propagated arguments") 208 | } 209 | } 210 | 211 | if !found { 212 | t.Fatalf("Kubelet doh should have directly configured extra arguments") 213 | } 214 | } 215 | 216 | func TestPoolPKIIntegration(t *testing.T) { 217 | t.Parallel() 218 | 219 | testPKI := &pki.PKI{ 220 | Kubernetes: &pki.Kubernetes{}, 221 | } 222 | 223 | if err := testPKI.Generate(); err != nil { 224 | t.Fatalf("Generating PKI: %v", err) 225 | } 226 | 227 | pool := &kubelet.Pool{ 228 | PKI: testPKI, 229 | AdminConfig: &client.Config{ 230 | Server: "foo", 231 | }, 232 | BootstrapConfig: &client.Config{ 233 | Server: "bar", 234 | Token: "bar", 235 | }, 236 | WaitForNodeReady: true, 237 | Kubelets: []kubelet.Kubelet{ 238 | { 239 | Name: "foo", 240 | VolumePluginDir: "foo", 241 | }, 242 | }, 243 | PrivilegedLabels: map[string]string{ 244 | "foo": "bar", 245 | }, 246 | } 247 | 248 | if _, err := pool.New(); err != nil { 249 | t.Fatalf("Creating kubelet pool with PKI integration should work, got: %v", err) 250 | } 251 | } 252 | 253 | func TestPoolNoKubelets(t *testing.T) { 254 | t.Parallel() 255 | 256 | testPKI := &pki.PKI{ 257 | Kubernetes: &pki.Kubernetes{}, 258 | } 259 | 260 | if err := testPKI.Generate(); err != nil { 261 | t.Fatalf("Generating PKI: %v", err) 262 | } 263 | 264 | pool := &kubelet.Pool{ 265 | PKI: testPKI, 266 | BootstrapConfig: &client.Config{ 267 | Server: "bar", 268 | Token: "bar", 269 | }, 270 | } 271 | 272 | if _, err := pool.New(); err == nil { 273 | t.Fatal("Creating kubelet pool with no kubelets and no state defined should fail") 274 | } 275 | } 276 | -------------------------------------------------------------------------------- /pkg/controlplane/kube-apiserver_test.go: -------------------------------------------------------------------------------- 1 | package controlplane 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/flexkube/libflexkube/internal/utiltest" 8 | "github.com/flexkube/libflexkube/pkg/host" 9 | "github.com/flexkube/libflexkube/pkg/host/transport/direct" 10 | "github.com/flexkube/libflexkube/pkg/types" 11 | ) 12 | 13 | const ( 14 | // TLS port used for testing. 15 | securePort = 6443 16 | 17 | // Non empty string used for testing. 18 | nonEmptyString = "foo" 19 | ) 20 | 21 | func TestKubeAPIServerToHostConfiguredContainer(t *testing.T) { 22 | t.Parallel() 23 | 24 | cert := types.Certificate(utiltest.GenerateX509Certificate(t)) 25 | privateKey := types.PrivateKey(utiltest.GenerateRSAPrivateKey(t)) 26 | 27 | kas := &KubeAPIServer{ 28 | Common: &Common{ 29 | KubernetesCACertificate: cert, 30 | FrontProxyCACertificate: cert, 31 | }, 32 | APIServerCertificate: cert, 33 | APIServerKey: privateKey, 34 | ServiceAccountPrivateKey: nonEmptyString, 35 | BindAddress: nonEmptyString, 36 | AdvertiseAddress: nonEmptyString, 37 | EtcdServers: []string{nonEmptyString}, 38 | ServiceCIDR: nonEmptyString, 39 | SecurePort: securePort, 40 | FrontProxyCertificate: cert, 41 | FrontProxyKey: privateKey, 42 | KubeletClientCertificate: cert, 43 | KubeletClientKey: privateKey, 44 | EtcdCACertificate: cert, 45 | EtcdClientCertificate: cert, 46 | EtcdClientKey: privateKey, 47 | Host: &host.Host{ 48 | DirectConfig: &direct.Config{}, 49 | }, 50 | } 51 | 52 | o, err := kas.New() 53 | if err != nil { 54 | t.Fatalf("New should not return error, got: %v", err) 55 | } 56 | 57 | hcc, err := o.ToHostConfiguredContainer() 58 | if err != nil { 59 | t.Fatalf("Generating HostConfiguredContainer should work, got: %v", err) 60 | } 61 | 62 | if _, err := hcc.New(); err != nil { 63 | t.Fatalf("ToHostConfiguredContainer() should generate valid HostConfiguredContainer, got: %v", err) 64 | } 65 | 66 | if hcc.Container.Config.Image == "" { 67 | t.Fatalf("New() should set default image if it's not present") 68 | } 69 | } 70 | 71 | func validKubeAPIServer(t *testing.T) *KubeAPIServer { 72 | t.Helper() 73 | 74 | cert := types.Certificate(utiltest.GenerateX509Certificate(t)) 75 | privateKey := types.PrivateKey(utiltest.GenerateRSAPrivateKey(t)) 76 | 77 | hostConfig := &host.Host{ 78 | DirectConfig: &direct.Config{}, 79 | } 80 | 81 | common := &Common{ 82 | KubernetesCACertificate: cert, 83 | FrontProxyCACertificate: cert, 84 | } 85 | 86 | return &KubeAPIServer{ 87 | Common: common, 88 | APIServerCertificate: cert, 89 | APIServerKey: privateKey, 90 | ServiceAccountPrivateKey: nonEmptyString, 91 | BindAddress: nonEmptyString, 92 | AdvertiseAddress: nonEmptyString, 93 | EtcdServers: []string{nonEmptyString}, 94 | ServiceCIDR: nonEmptyString, 95 | SecurePort: securePort, 96 | FrontProxyCertificate: cert, 97 | FrontProxyKey: privateKey, 98 | KubeletClientKey: privateKey, 99 | EtcdCACertificate: cert, 100 | EtcdClientCertificate: cert, 101 | EtcdClientKey: privateKey, 102 | Host: hostConfig, 103 | KubeletClientCertificate: cert, 104 | } 105 | } 106 | 107 | // Validate() tests. 108 | func TestKubeAPIServerValidate(t *testing.T) { 109 | t.Parallel() 110 | 111 | cases := map[string]struct { 112 | MutateF func(*KubeAPIServer) 113 | Error bool 114 | }{ 115 | "require kubeletClientCertificate": { 116 | MutateF: func(k *KubeAPIServer) { 117 | k.KubeletClientCertificate = "" 118 | }, 119 | Error: true, 120 | }, 121 | "validate kubeletClientCertificate": { 122 | MutateF: func(k *KubeAPIServer) { 123 | k.KubeletClientCertificate = nonEmptyString 124 | }, 125 | Error: true, 126 | }, 127 | "require at least one etcd server": { 128 | MutateF: func(k *KubeAPIServer) { 129 | k.EtcdServers = []string{} 130 | }, 131 | Error: true, 132 | }, 133 | "validate host": { 134 | MutateF: func(k *KubeAPIServer) { 135 | k.Host = &host.Host{} 136 | }, 137 | Error: true, 138 | }, 139 | "valid": { 140 | MutateF: func(_ *KubeAPIServer) {}, 141 | Error: false, 142 | }, 143 | } 144 | 145 | for n, testCase := range cases { 146 | testCase := testCase 147 | 148 | t.Run(n, func(t *testing.T) { 149 | t.Parallel() 150 | 151 | config := validKubeAPIServer(t) 152 | testCase.MutateF(config) 153 | 154 | err := config.Validate() 155 | if !testCase.Error && err != nil { 156 | t.Errorf("Didn't expect error, got: %v", err) 157 | } 158 | 159 | if testCase.Error && err == nil { 160 | t.Errorf("Expected error") 161 | } 162 | }) 163 | } 164 | } 165 | 166 | func TestKubeAPIServerConfigFiles(t *testing.T) { 167 | t.Parallel() 168 | 169 | cert := types.Certificate(utiltest.GenerateX509Certificate(t)) 170 | privateKey := types.PrivateKey(utiltest.GenerateRSAPrivateKey(t)) 171 | 172 | hostConfig := &host.Host{ 173 | DirectConfig: &direct.Config{}, 174 | } 175 | 176 | common := &Common{ 177 | KubernetesCACertificate: cert, 178 | FrontProxyCACertificate: cert, 179 | } 180 | 181 | testConfig := &KubeAPIServer{ 182 | Common: common, 183 | APIServerCertificate: cert, 184 | APIServerKey: privateKey, 185 | ServiceAccountPrivateKey: nonEmptyString, 186 | BindAddress: nonEmptyString, 187 | AdvertiseAddress: nonEmptyString, 188 | EtcdServers: []string{nonEmptyString}, 189 | ServiceCIDR: nonEmptyString, 190 | SecurePort: securePort, 191 | FrontProxyCertificate: cert, 192 | FrontProxyKey: privateKey, 193 | KubeletClientKey: privateKey, 194 | EtcdCACertificate: cert, 195 | EtcdClientCertificate: cert, 196 | EtcdClientKey: privateKey, 197 | Host: hostConfig, 198 | KubeletClientCertificate: cert, 199 | } 200 | 201 | ki, err := testConfig.New() 202 | if err != nil { 203 | t.Fatalf("KubeAPIServer object should be created, got: %v", err) 204 | } 205 | 206 | hcc, err := ki.ToHostConfiguredContainer() 207 | if err != nil { 208 | t.Fatalf("Converting kube-apiserver to host configured container: %v", err) 209 | } 210 | 211 | for k := range hcc.ConfigFiles { 212 | if !strings.Contains(k, hostConfigPath) { 213 | t.Fatalf("All config files paths should contain %s, got: %s", hostConfigPath, k) 214 | } 215 | } 216 | } 217 | 218 | // New() tests. 219 | func TestKubeAPIServerNewEmptyHost(t *testing.T) { 220 | t.Parallel() 221 | 222 | c := &KubeAPIServer{} 223 | 224 | k, err := c.New() 225 | if err == nil { 226 | t.Errorf("New on empty config should return error") 227 | } 228 | 229 | if k != nil { 230 | t.Errorf("New should not return kube-apiserver object in case of error") 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /pkg/container/container_integration_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package container 5 | 6 | import ( 7 | "crypto/rand" 8 | "crypto/sha256" 9 | "fmt" 10 | "testing" 11 | 12 | "github.com/flexkube/libflexkube/pkg/container/runtime/docker" 13 | "github.com/flexkube/libflexkube/pkg/container/types" 14 | "github.com/flexkube/libflexkube/pkg/defaults" 15 | ) 16 | 17 | // Create() tests. 18 | func TestDockerCreateNonExistingImage(t *testing.T) { 19 | t.Parallel() 20 | 21 | containerConfig := &Container{ 22 | Runtime: RuntimeConfig{ 23 | Docker: &docker.Config{}, 24 | }, 25 | Config: types.ContainerConfig{ 26 | Name: "foo", 27 | Image: "notexisting", 28 | }, 29 | } 30 | 31 | c, err := containerConfig.New() 32 | if err != nil { 33 | t.Fatalf("Initializing container should succeed, got: %v", err) 34 | } 35 | 36 | if _, err = c.Create(); err == nil { 37 | t.Fatalf("Creating container with non-existing image should fail") 38 | } 39 | } 40 | 41 | func TestDockerCreate(t *testing.T) { 42 | t.Parallel() 43 | 44 | containerConfig := &Container{ 45 | Runtime: RuntimeConfig{ 46 | Docker: &docker.Config{}, 47 | }, 48 | Config: types.ContainerConfig{ 49 | Name: randomContainerName(t), 50 | Image: defaults.EtcdImage, 51 | }, 52 | } 53 | 54 | c, err := containerConfig.New() 55 | if err != nil { 56 | t.Fatalf("Initializing container should succeed, got: %v", err) 57 | } 58 | 59 | containerID, err := c.Create() 60 | if err != nil { 61 | t.Fatalf("Creating container should succeed, got: %v", err) 62 | } 63 | 64 | t.Cleanup(func() { 65 | if err := containerID.Delete(); err != nil { 66 | t.Logf("Removing container should succeed, got: %v", err) 67 | } 68 | }) 69 | } 70 | 71 | // Status() tests. 72 | func TestDockerStatus(t *testing.T) { 73 | t.Parallel() 74 | 75 | containerConfig := &Container{ 76 | Runtime: RuntimeConfig{ 77 | Docker: &docker.Config{}, 78 | }, 79 | Config: types.ContainerConfig{ 80 | Name: randomContainerName(t), 81 | Image: defaults.EtcdImage, 82 | }, 83 | } 84 | 85 | c, err := containerConfig.New() 86 | if err != nil { 87 | t.Fatalf("Initializing container should succeed, got: %v", err) 88 | } 89 | 90 | containerID, err := c.Create() 91 | if err != nil { 92 | t.Fatalf("Creating container should succeed, got: %v", err) 93 | } 94 | 95 | if _, err := containerID.Status(); err != nil { 96 | t.Fatalf("Checking container status should succeed, got: %v", err) 97 | } 98 | 99 | t.Cleanup(func() { 100 | if err := containerID.Delete(); err != nil { 101 | t.Logf("Removing container should succeed, got: %v", err) 102 | } 103 | }) 104 | } 105 | 106 | func TestDockerStatusNonExistingContainer(t *testing.T) { 107 | t.Parallel() 108 | 109 | containerConfig := &Container{ 110 | Runtime: RuntimeConfig{ 111 | Docker: &docker.Config{}, 112 | }, 113 | Config: types.ContainerConfig{ 114 | Name: randomContainerName(t), 115 | Image: defaults.EtcdImage, 116 | }, 117 | } 118 | 119 | c, err := containerConfig.New() 120 | if err != nil { 121 | t.Fatalf("Initializing container should succeed, got: %v", err) 122 | } 123 | 124 | containerID, err := c.Create() 125 | if err != nil { 126 | t.Fatalf("Creating container should succeed, got: %v", err) 127 | } 128 | 129 | testContainerInstance, ok := containerID.(*containerInstance) 130 | if !ok { 131 | t.Fatalf("Unexpected type for containerID: %T", containerID) 132 | } 133 | 134 | originalCIID := testContainerInstance.status.ID 135 | 136 | testContainerInstance.status.ID = "" 137 | 138 | status, err := containerID.Status() 139 | if err != nil { 140 | t.Fatalf("Checking container status for non existing container should succeed") 141 | } 142 | 143 | if status.ID != "" { 144 | t.Fatalf("Container ID for non existing container should be empty") 145 | } 146 | 147 | testContainerInstance.status.ID = originalCIID 148 | 149 | t.Cleanup(func() { 150 | if err := containerID.Delete(); err != nil { 151 | t.Logf("Removing container should succeed, got: %v", err) 152 | } 153 | }) 154 | } 155 | 156 | // Start() tests. 157 | func TestDockerStart(t *testing.T) { 158 | t.Parallel() 159 | 160 | containerConfig := &Container{ 161 | Runtime: RuntimeConfig{ 162 | Docker: &docker.Config{}, 163 | }, 164 | Config: types.ContainerConfig{ 165 | Name: randomContainerName(t), 166 | Image: defaults.EtcdImage, 167 | }, 168 | } 169 | 170 | c, err := containerConfig.New() 171 | if err != nil { 172 | t.Fatalf("Initializing container should succeed, got: %v", err) 173 | } 174 | 175 | containerID, err := c.Create() 176 | if err != nil { 177 | t.Fatalf("Creating container should succeed, got: %v", err) 178 | } 179 | 180 | if err := containerID.Start(); err != nil { 181 | t.Fatalf("Starting container should succeed, got: %v", err) 182 | } 183 | 184 | t.Cleanup(func() { 185 | if err := containerID.Stop(); err != nil { 186 | t.Logf("Stopping container should succeed, got: %v", err) 187 | 188 | // Deleting not stopped container will fail, so return early. 189 | return 190 | } 191 | 192 | if err := containerID.Delete(); err != nil { 193 | t.Logf("Removing container should succeed, got: %v", err) 194 | } 195 | }) 196 | } 197 | 198 | // Stop() tests. 199 | func TestDockerStop(t *testing.T) { 200 | t.Parallel() 201 | 202 | containerConfig := &Container{ 203 | Runtime: RuntimeConfig{ 204 | Docker: &docker.Config{}, 205 | }, 206 | Config: types.ContainerConfig{ 207 | Name: randomContainerName(t), 208 | Image: defaults.EtcdImage, 209 | }, 210 | } 211 | 212 | c, err := containerConfig.New() 213 | if err != nil { 214 | t.Fatalf("Initializing container should succeed, got: %v", err) 215 | } 216 | 217 | containerID, err := c.Create() 218 | if err != nil { 219 | t.Fatalf("Creating container should succeed, got: %v", err) 220 | } 221 | 222 | if err := containerID.Start(); err != nil { 223 | t.Fatalf("Starting container should succeed, got: %v", err) 224 | } 225 | 226 | if err := containerID.Stop(); err != nil { 227 | t.Fatalf("Stopping container should succeed, got: %v", err) 228 | } 229 | 230 | t.Cleanup(func() { 231 | if err := containerID.Delete(); err != nil { 232 | t.Logf("Removing container should succeed, got: %v", err) 233 | } 234 | }) 235 | } 236 | 237 | // Delete() tests. 238 | func TestDockerDelete(t *testing.T) { 239 | t.Parallel() 240 | 241 | containerConfig := &Container{ 242 | Runtime: RuntimeConfig{ 243 | Docker: &docker.Config{}, 244 | }, 245 | Config: types.ContainerConfig{ 246 | Name: randomContainerName(t), 247 | Image: defaults.EtcdImage, 248 | }, 249 | } 250 | 251 | c, err := containerConfig.New() 252 | if err != nil { 253 | t.Fatalf("Initializing container should succeed, got: %v", err) 254 | } 255 | 256 | containerID, err := c.Create() 257 | if err != nil { 258 | t.Fatalf("Creating container should succeed, got: %v", err) 259 | } 260 | 261 | if err := containerID.Delete(); err != nil { 262 | t.Fatalf("Removing container should succeed, got: %v", err) 263 | } 264 | } 265 | 266 | func randomContainerName(t *testing.T) string { 267 | t.Helper() 268 | 269 | token := make([]byte, 32) 270 | 271 | if _, err := rand.Read(token); err != nil { 272 | t.Fatalf("Generating random container name: %v", err) 273 | } 274 | 275 | return fmt.Sprintf("foo-%x", sha256.Sum256(token)) 276 | } 277 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Certified Kubernetes logo 2 | Flexkube logo 3 | 4 | # libflexkube: Go library for deploying Kubernetes 5 | 6 | [![Build Status](https://travis-ci.org/flexkube/libflexkube.svg?branch=master)](https://travis-ci.org/flexkube/libflexkube) [![Maintainability](https://api.codeclimate.com/v1/badges/5840c3fe0a9bc77aef08/maintainability)](https://codeclimate.com/github/flexkube/libflexkube/maintainability) [![codecov](https://codecov.io/gh/flexkube/libflexkube/branch/master/graph/badge.svg)](https://codecov.io/gh/flexkube/libflexkube) [![GoDoc](https://godoc.org/github.com/flexkube/libflexkube?status.svg)](https://godoc.org/github.com/flexkube/libflexkube) [![Go Report Card](https://goreportcard.com/badge/github.com/flexkube/libflexkube)](https://goreportcard.com/report/github.com/flexkube/libflexkube) 7 | 8 | ## Table of contents 9 | 10 | - [Introduction](#introduction) 11 | - [Documentation](#documentation) 12 | - [Installation and usage](#installation-and-usage) 13 | - [CLI tool](#cli-tool) 14 | - [Terraform](#terraform) 15 | - [Next steps](#next-steps) 16 | - [Characteristics](#characteristics) 17 | - [Features](#features) 18 | - [Known issues, missing features and limitations](#known-issues-missing-features-and-limitations) 19 | - [Contributing](#contributing) 20 | - [Status of the project](#status-of-the-project) 21 | 22 | ## Introduction 23 | 24 | libflexkube is the core part of Flexkube project. It is a Go library, which implements the logic for managing Kubernetes cluster components (e.g. [etcd](https://etcd.io/), [kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/), [certificates](https://kubernetes.io/docs/setup/best-practices/certificates/)) and provides reference implementation of [flexkube](https://flexkube.github.io/documentation/reference/cli/) CLI tool and [Terraform provider](https://registry.terraform.io/providers/flexkube/flexkube/latest/docs). 25 | 26 | Flexkube is a minimalistic and flexible Kubernetes distribution, providing tools to manage each main Kubernetes component independently, which can be used together to create Kubernetes clusters. 27 | 28 | ## Documentation 29 | 30 | To see user documentation, visit [flexkube.github.io](https://flexkube.github.io). Please note, that the documentation is still in progress. 31 | 32 | For development documentation, see [docs](docs). 33 | 34 | ## Installation and usage 35 | 36 | For quick examples of installation and usage, see the content below. For full documentation, see [Getting started](https://flexkube.github.io/documentation/getting-started/). 37 | 38 | ### CLI tool 39 | 40 | If you have `go` binary available in your system, you can start using Flexkube for creating Kubernetes certificates just by running the following commands: 41 | ```sh 42 | echo 'pki: 43 | kubernetes: {} 44 | etcd: {}' > config.yaml 45 | go run github.com/flexkube/libflexkube/cmd/flexkube@latest pki 46 | ``` 47 | 48 | It will create `config.yaml` configuration file which will be consumed by `flexkube` CLI tool, which will then generate the certificates into `state.yaml` file. 49 | 50 | ### Terraform 51 | 52 | If you want to perform the same action using Terraform, execute the following commands: 53 | 54 | ```sh 55 | cat < main.tf 56 | terraform { 57 | required_providers { 58 | flexkube = { 59 | source = "flexkube/flexkube" 60 | version = "0.9.0" 61 | } 62 | } 63 | } 64 | 65 | resource "flexkube_pki" "pki" { 66 | etcd {} 67 | kubernetes {} 68 | } 69 | EOF 70 | terraform init && terraform apply 71 | ``` 72 | 73 | After applying, Kubernetes certificates should be available as Terraform resource attributes of `flexkube_pki.pki` resource. 74 | 75 | ### Next steps 76 | 77 | For more detailed instructions of how to use Flexkube, see the user [guides](https://flexkube.github.io/documentation/guides). 78 | 79 | ## Characteristics 80 | 81 | Flexkube project focuses on simplicity and tries to only do minimal amount of steps in order to get Kubernetes cluster running, while keeping the configuration flexible and extensible. It is also a good material for learning how Kubernetes cluster is set up, as each part of the cluster is managed independently and code contains a lot of comments why specific flag/option is needed and what purpose does it serve. 82 | 83 | Parts of this project could possibly be used in other Kubernetes distributions or be used as a configuration reference, as setting up Kubernetes components requires many flags and configuration files to be created. 84 | 85 | Flexkube do not manage infrastructure for running Kubernetes clusters, it must be provided by the user. 86 | 87 | ## Features 88 | 89 | Here is the short list of Flexkube project features: 90 | 91 | - Minimal host requirements - Use SSH connection forwarding for talking directly to the container runtime on remote machines for managing static containers and configuration files. 92 | - Independent management of etcd, kubelets, static control plane and self-hosted components. 93 | - All self-hosted control plane components managed using Helm 3 (e.g CoreDNS). 94 | - 1st class support for Terraform provider for automation. 95 | - No Public DNS or any other public discovery service is required for getting cluster up and running. 96 | - Others: 97 | - etcd, kubelet and static control plane running as containers. 98 | - Self-hosted control plane. 99 | - Supported container runtimes: 100 | - Docker 101 | - Configuration via YAML or via Terraform. 102 | - Deployment using CLI tools or via Terraform. 103 | - HAProxy for load-balancing and fail-over between Kubernetes API servers. 104 | 105 | ## Known issues, missing features and limitations 106 | 107 | As the project is still in the early stages, here is the list of major existing issues or missing features, which are likely to be implemented in the future: 108 | 109 | - gracefully replacing CA certificates (if private key does not change, it should work, but has not been tested) 110 | - no checkpointer for pods/apiserver. If static kube-apiserver container is stopped and node reboots, single node cluster will not come back. 111 | 112 | And features, which are not yet implemented: 113 | 114 | - network policies for kube-system namespace 115 | - caching port forwarding 116 | - bastion host(s) support for SSH 117 | - parallel deployments across hosts 118 | - removal of configuration files, created data and containers 119 | - automatic shutdown/start of bootstrap control plane 120 | 121 | ## Contributing 122 | 123 | All contributions to this project are welcome. If it does not satisfy your needs, feel free to raise an issue about it or implement the support yourself and create a pull request with the patch, so we can all benefit from it. 124 | 125 | If you just want to help the project grow and mature, there are many TODOs spread across the code, which should be addresses sooner or later. 126 | 127 | ## Status of the project 128 | 129 | At the time of writing, this project is in active development state and it is not suitable for production use. Breaking changes might be introduced at any point in time, for both library interface and for existing deployments. 130 | --------------------------------------------------------------------------------