├── CODEOWNERS ├── scripts ├── release ├── ci ├── entry ├── build ├── test ├── validate ├── version └── package ├── .gitignore ├── .dockerignore ├── nginx-proxy ├── conf.d └── nginx.toml ├── .github ├── renovate.json └── workflows │ ├── fossa.yaml │ ├── renovate.yml │ └── workflow.yaml ├── weave-plugins-cni.sh ├── share-root.sh ├── manifest.tmpl ├── Makefile ├── templates └── nginx.tmpl ├── Dockerfile.dapper ├── go.mod ├── README.md ├── cert-deployer ├── package └── Dockerfile ├── docs └── components.md ├── go.sum ├── cloud-provider.sh ├── entrypoint.sh ├── LICENSE └── main.go /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @rancher/rancher-team-2-hostbusters-dev 2 | -------------------------------------------------------------------------------- /scripts/release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exec $(dirname $0)/ci 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.dapper 2 | /bin 3 | /dist 4 | *.swp 5 | /.idea 6 | 7 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | ./bin 2 | !bin/rke-etcd-backup 3 | ./.dapper 4 | ./dist 5 | 6 | -------------------------------------------------------------------------------- /nginx-proxy: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Run confd 4 | confd -onetime -backend env 5 | 6 | # Start nginx 7 | nginx -g 'daemon off;' 8 | -------------------------------------------------------------------------------- /conf.d/nginx.toml: -------------------------------------------------------------------------------- 1 | [template] 2 | src = "nginx.tmpl" 3 | dest = "/etc/nginx/nginx.conf" 4 | keys = [ 5 | "CP_HOSTS", 6 | ] 7 | -------------------------------------------------------------------------------- /scripts/ci: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd $(dirname $0) 5 | 6 | ./build 7 | ./test 8 | ./validate 9 | ./package 10 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "github>rancher/renovate-config#release" 4 | ], 5 | "baseBranches": [ 6 | "master" 7 | ], 8 | "prHourlyLimit": 2 9 | } 10 | -------------------------------------------------------------------------------- /scripts/entry: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | mkdir -p bin dist 5 | if [ -e ./scripts/$1 ]; then 6 | ./scripts/"$@" 7 | else 8 | exec "$@" 9 | fi 10 | 11 | chown -R $DAPPER_UID:$DAPPER_GID . 12 | -------------------------------------------------------------------------------- /weave-plugins-cni.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | # deploy loopback cni 4 | mkdir -p /opt/cni/bin 5 | mv /tmp/loopback /opt/cni/bin 6 | mv /tmp/portmap /opt/cni/bin 7 | chmod 755 /opt/cni/bin/portmap 8 | while true; do 9 | sleep 100 10 | done 11 | -------------------------------------------------------------------------------- /share-root.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ID=$(grep :devices: /proc/self/cgroup | head -n1 | awk -F/ '{print $NF}') 4 | IMAGE=$(docker inspect -f '{{.Config.Image}}' $ID) 5 | 6 | docker run --privileged --net host --pid host -v /:/host --rm --entrypoint /usr/bin/share-mnt $IMAGE "$@" -- norun 7 | -------------------------------------------------------------------------------- /scripts/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/version 5 | 6 | cd $(dirname $0)/.. 7 | 8 | mkdir -p bin 9 | [ "$(uname)" != "Darwin" ] && LINKFLAGS="-extldflags -static -s" 10 | CGO_ENABLED=0 go build -ldflags "-X main.VERSION=$VERSION $LINKFLAGS" -o bin/rke-etcd-backup . 11 | -------------------------------------------------------------------------------- /manifest.tmpl: -------------------------------------------------------------------------------- 1 | image: rancher/rke-tools:{{build.tag}} 2 | manifests: 3 | - 4 | image: rancher/rke-tools:{{build.tag}}-linux-amd64 5 | platform: 6 | architecture: amd64 7 | os: linux 8 | - 9 | image: rancher/rke-tools:{{build.tag}}-linux-arm64 10 | platform: 11 | architecture: arm64 12 | os: linux 13 | -------------------------------------------------------------------------------- /scripts/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd $(dirname $0)/.. 5 | 6 | echo Running tests 7 | 8 | PACKAGES=". $(find -name '*.go' | xargs -I{} dirname {} | cut -f2 -d/ | sort -u | grep -Ev '(^\.$|.git|vendor|bin)' | sed -e 's!^!./!' -e 's!$!/...!')" 9 | 10 | [ "${ARCH}" == "amd64" ] && RACE=-race 11 | go test ${RACE} -cover -tags=test ${PACKAGES} 12 | -------------------------------------------------------------------------------- /scripts/validate: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd "$(dirname $0)/.." 5 | 6 | echo Running validation 7 | 8 | PACKAGES="$(go list ./...)" 9 | 10 | echo Running: go vet 11 | go vet "${PACKAGES}" 12 | 13 | echo Running: golangci-lint 14 | golangci-lint run --disable-all -E revive 15 | 16 | echo Running: go fmt 17 | test -z "$(go fmt ${PACKAGES} | tee /dev/stderr)" 18 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TARGETS := $(shell ls scripts) 2 | 3 | .dapper: 4 | @echo Downloading dapper 5 | @curl -sL https://releases.rancher.com/dapper/latest/dapper-`uname -s`-`uname -m` > .dapper.tmp 6 | @@chmod +x .dapper.tmp 7 | @./.dapper.tmp -v 8 | @mv .dapper.tmp .dapper 9 | 10 | $(TARGETS): .dapper 11 | ./.dapper $@ 12 | 13 | .DEFAULT_GOAL := ci 14 | 15 | .PHONY: $(TARGETS) 16 | -------------------------------------------------------------------------------- /scripts/version: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -n "$(git status --porcelain --untracked-files=no)" ]; then 4 | DIRTY="-dirty" 5 | fi 6 | 7 | COMMIT=$(git rev-parse --short HEAD) 8 | 9 | if [[ $GITHUB_ACTIONS = true && $GITHUB_REF_TYPE = "tag" ]]; then 10 | TAG_NAME=$GITHUB_REF_NAME 11 | fi 12 | 13 | GIT_TAG=${TAG_NAME:-$(git tag -l --contains HEAD | head -n 1)} 14 | 15 | if [[ -z "$DIRTY" && -n "$GIT_TAG" ]]; then 16 | VERSION=$GIT_TAG 17 | else 18 | VERSION="${COMMIT}${DIRTY}" 19 | fi 20 | 21 | if [ -z "$ARCH" ]; then 22 | ARCH=amd64 23 | fi 24 | -------------------------------------------------------------------------------- /templates/nginx.tmpl: -------------------------------------------------------------------------------- 1 | error_log stderr notice; 2 | 3 | worker_processes auto; 4 | events { 5 | multi_accept on; 6 | use epoll; 7 | worker_connections 1024; 8 | } 9 | 10 | stream { 11 | upstream kube_apiserver { 12 | {{ $servers := split (getenv "CP_HOSTS") "," }}{{range $servers}} 13 | server {{.}}:6443; 14 | {{end}} 15 | } 16 | 17 | server { 18 | listen 6443; 19 | proxy_pass kube_apiserver; 20 | proxy_timeout 10m; 21 | proxy_connect_timeout 2s; 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /scripts/package: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/version 5 | 6 | ARCH=${ARCH:-"amd64"} 7 | SUFFIX="" 8 | [ "${ARCH}" != "amd64" ] && SUFFIX="_${ARCH}" 9 | 10 | cd $(dirname $0)/../package 11 | 12 | TAG=${TAG:-${VERSION}${SUFFIX}} 13 | REPO=${REPO:-rancher} 14 | 15 | if echo $TAG | grep -q dirty; then 16 | TAG=dev 17 | fi 18 | 19 | cp ../bin/rke-etcd-backup . 20 | cp -r ../{conf.d,cert-deployer,templates,nginx-proxy,*.sh} . 21 | 22 | IMAGE=${REPO}/rke-tools:${TAG} 23 | docker build --build-arg ARCH=${ARCH} -t ${IMAGE} . 24 | mkdir -p ../dist 25 | echo ${IMAGE} > ../dist/images 26 | echo Built ${IMAGE} 27 | -------------------------------------------------------------------------------- /.github/workflows/fossa.yaml: -------------------------------------------------------------------------------- 1 | name: Fossa scan 2 | 3 | on: 4 | push: 5 | branches: 6 | - '*' 7 | tags: 8 | - '*' 9 | 10 | jobs: 11 | fossa: 12 | runs-on: ubuntu-latest 13 | permissions: 14 | contents: read 15 | id-token: write 16 | timeout-minutes: 20 17 | steps: 18 | - name: Checkout Repo 19 | uses: actions/checkout@v4 20 | - name: Read FOSSA token 21 | uses: rancher-eio/read-vault-secrets@main 22 | with: 23 | secrets: | 24 | secret/data/github/org/rancher/fossa/push token | FOSSA_API_KEY_PUSH_ONLY 25 | - name: FOSSA scan 26 | uses: fossas/fossa-action@main 27 | with: 28 | api-key: ${{ env.FOSSA_API_KEY_PUSH_ONLY }} 29 | run-tests: false -------------------------------------------------------------------------------- /Dockerfile.dapper: -------------------------------------------------------------------------------- 1 | FROM registry.suse.com/bci/golang:1.23 2 | 3 | ARG DAPPER_HOST_ARCH 4 | ENV HOST_ARCH=${DAPPER_HOST_ARCH} ARCH=${DAPPER_HOST_ARCH} 5 | 6 | RUN zypper -n install gcc vim less file docker git wget curl go 7 | 8 | ENV GOLANG_ARCH_amd64=amd64 GOLANG_ARCH_arm=armv6l GOLANG_ARCH_arm64=arm64 GOLANG_ARCH=GOLANG_ARCH_${ARCH} \ 9 | GOPATH=/go PATH=/go/bin:/usr/local/go/bin:${PATH} SHELL=/bin/bash 10 | 11 | ENV GOLANGCI_LINT=v1.59.1 12 | RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s ${GOLANGCI_LINT}; 13 | 14 | ENV DAPPER_ENV REPO TAG DRONE_TAG 15 | ENV DAPPER_SOURCE /go/src/github.com/rancher/rke-tools/ 16 | ENV DAPPER_OUTPUT ./bin ./dist 17 | ENV DAPPER_DOCKER_SOCKET true 18 | ENV HOME ${DAPPER_SOURCE} 19 | WORKDIR ${DAPPER_SOURCE} 20 | 21 | ENTRYPOINT ["./scripts/entry"] 22 | CMD ["ci"] 23 | -------------------------------------------------------------------------------- /.github/workflows/renovate.yml: -------------------------------------------------------------------------------- 1 | name: Renovate 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | logLevel: 6 | description: "Override default log level" 7 | required: false 8 | default: "info" 9 | type: string 10 | overrideSchedule: 11 | description: "Override all schedules" 12 | required: false 13 | default: "false" 14 | type: string 15 | # Run twice in the early morning (UTC) for initial and follow up steps (create pull request and merge) 16 | schedule: 17 | - cron: '30 4,6 * * *' 18 | 19 | jobs: 20 | call-workflow: 21 | uses: rancher/renovate-config/.github/workflows/renovate.yml@release 22 | with: 23 | logLevel: ${{ inputs.logLevel || 'info' }} 24 | overrideSchedule: ${{ github.event.inputs.overrideSchedule == 'true' && '{''schedule'':null}' || '' }} 25 | secrets: inherit 26 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rancher/rke-tools 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.6 6 | 7 | require ( 8 | github.com/minio/minio-go/v7 v7.0.74 9 | github.com/sirupsen/logrus v1.9.3 10 | github.com/urfave/cli v1.22.15 11 | ) 12 | 13 | require ( 14 | github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect 15 | github.com/dustin/go-humanize v1.0.1 // indirect 16 | github.com/go-ini/ini v1.67.0 // indirect 17 | github.com/goccy/go-json v0.10.3 // indirect 18 | github.com/google/uuid v1.6.0 // indirect 19 | github.com/klauspost/compress v1.17.9 // indirect 20 | github.com/klauspost/cpuid/v2 v2.2.8 // indirect 21 | github.com/minio/md5-simd v1.1.2 // indirect 22 | github.com/rs/xid v1.5.0 // indirect 23 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 24 | golang.org/x/crypto v0.24.0 // indirect 25 | golang.org/x/net v0.26.0 // indirect 26 | golang.org/x/sys v0.21.0 // indirect 27 | golang.org/x/text v0.16.0 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rke-tools 2 | 3 | ## About 4 | 5 | The container image `rancher/rke-tools` is used in Kubernetes clusters built by RKE (`rancher/rke`) as: 6 | 7 | - Entrypoint for each k8s container created by RKE (`entrypoint.sh`, `cloud-provider.sh`) 8 | - Container to operate etcd snapshots in RKE clusters (create/remove/restore) (`main.go`) 9 | - Container to run the proxy between `kubelet` and `kube-apiserver` in RKE clusters (`nginx-proxy`, `conf.d/nginx.toml`) 10 | - Container to deploy Kubernetes certificates needed by the nodes in RKE clusters (`cert-deployer`) 11 | - Container to deploy Weave loopback/portmap plugin (`weave-plugins-cni.sh`) 12 | - Container to make mounts shared (`share-root.sh`, deprecated, used for boot2docker) 13 | 14 | See [components.md](./docs/components.md) for a more detailed explanation of each component. 15 | 16 | ## Building 17 | 18 | Running `make` should run the default target (`ci`), which runs all the scripts needed to built a binary and container. It uses `rancher/dapper` as build wrapper. You can run each steps separately if you want to skip some of the defaults, for example `make build`. 19 | 20 | ## Testing 21 | 22 | To test your newly built image, you need to make the image available on a Docker registry that is available to your RKE cluster nodes or import the image manually to your RKE cluster nodes (RKE will look for images locally available before pulling from the registry). Now you can use the following configuration in your `cluster.yml` file to use your new image in testing: 23 | 24 | ```yaml 25 | system_images: 26 | alpine: your_name/rke-tools:your_tag 27 | nginx_proxy: your_name/rke-tools:your_tag 28 | cert_downloader: your_name/rke-tools:your_tag 29 | kubernetes_services_sidecar: your_name/rke-tools:your_tag 30 | ``` 31 | -------------------------------------------------------------------------------- /cert-deployer: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | SSL_CRTS_DIR=${CRTS_DEPLOY_PATH:-/etc/kubernetes/ssl} 4 | mkdir -p $SSL_CRTS_DIR 5 | chmod 755 $SSL_CRTS_DIR 6 | 7 | for i in $(env | grep -o KUBE_.*=); do 8 | name="$(echo "$i" | cut -f1 -d"=" | tr '[:upper:]' '[:lower:]' | tr '_' '-').pem" 9 | env=$(echo "$i" | cut -f1 -d"=") 10 | value=$(echo "${!env}") 11 | if [ ! -f $SSL_CRTS_DIR/$name ] || [ "$FORCE_DEPLOY" = "true" ]; then 12 | echo "$value" > $SSL_CRTS_DIR/$name 13 | chmod 600 $SSL_CRTS_DIR/$name 14 | fi 15 | done 16 | 17 | for i in $(env | grep -o KUBECFG_.*=); do 18 | name="$(echo "$i" | cut -f1 -d"=" | tr '[:upper:]' '[:lower:]' | tr '_' '-').yaml" 19 | env=$(echo "$i" | cut -f1 -d"=") 20 | value=$(echo "${!env}") 21 | if [ ! -f $SSL_CRTS_DIR/$name ]; then 22 | echo "$value" > $SSL_CRTS_DIR/$name 23 | chmod 600 $SSL_CRTS_DIR/$name 24 | fi 25 | done 26 | 27 | # only enabled if we are running etcd with custom uid/gid 28 | # change ownership of etcd cert and key and kube-ca to the custom uid/gid 29 | if [ -n "${ETCD_UID}" ] && [ -n "${ETCD_GID}" ]; then 30 | # set minial mask to allow effective read access to the certificates 31 | setfacl -R -m m::rX "${SSL_CRTS_DIR}" && echo "Successfully set ACL mask for certs dir" 32 | # we remove certs dir acl if any for the custom etcd uid, since chown will give that access 33 | setfacl -R -x u:${ETCD_UID} "${SSL_CRTS_DIR}" && echo "Successfully unset user ACL for certs dir" 34 | # allow certs dir read access to the custom etcd gid 35 | setfacl -R -x g:${ETCD_GID} "${SSL_CRTS_DIR}" && echo "Successfully unset group ACL for certs dir" 36 | 37 | for name in $SSL_CRTS_DIR/*.pem; do 38 | if [[ $name == *kube-etcd* ]] ; then 39 | chown "${ETCD_UID}":"${ETCD_GID}" $name 40 | fi 41 | if [[ $name == *kube-ca.pem ]] ; then 42 | chmod 644 $name 43 | fi 44 | done 45 | chmod 755 $SSL_CRTS_DIR 46 | fi -------------------------------------------------------------------------------- /package/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rancher/hardened-cni-plugins:v1.5.1-build20240910 as cni_base 2 | 3 | FROM nginx:1.27.1-alpine as base 4 | 5 | ENV DOCKER_VERSION=27.1.1 6 | ENV ETCD_VERSION=v3.5.16 7 | ENV CRIDOCKERD_VERSION=0.3.16 8 | ENV RANCHER_CONFD_VERSION=v0.16.7 9 | ENV KUBECTL_VERSION=v1.28.13 10 | 11 | LABEL maintainer "Rancher Labs " 12 | ARG ARCH=amd64 13 | ENV DOCKER_URL_amd64="https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" \ 14 | DOCKER_URL_arm64="https://download.docker.com/linux/static/stable/aarch64/docker-${DOCKER_VERSION}.tgz" \ 15 | DOCKER_URL="DOCKER_URL_${ARCH}" 16 | ENV CRIDOCKERD_URL="https://github.com/Mirantis/cri-dockerd/releases/download/v${CRIDOCKERD_VERSION}/cri-dockerd-${CRIDOCKERD_VERSION}.${ARCH}.tgz" 17 | RUN apk -U upgrade \ 18 | && apk -U --no-cache add bash \ 19 | && rm -f /bin/sh \ 20 | && ln -s /bin/bash /bin/sh 21 | RUN apk -U --no-cache add curl wget ca-certificates tar sysstat acl\ 22 | && mkdir -p /opt/rke-tools/bin /etc/confd \ 23 | && curl -sLf "https://github.com/rancher/confd/releases/download/${RANCHER_CONFD_VERSION}/confd-${RANCHER_CONFD_VERSION}-linux-${ARCH}" > /usr/bin/confd \ 24 | && chmod +x /usr/bin/confd \ 25 | && curl -sLf "${!DOCKER_URL}" | tar xvzf - -C /opt/rke-tools/bin --strip-components=1 docker/docker \ 26 | && curl -sLf "${CRIDOCKERD_URL}" | tar xvzf - -C /opt/rke-tools/bin --strip-components=1 cri-dockerd/cri-dockerd \ 27 | && chmod +x /opt/rke-tools/bin/cri-dockerd \ 28 | && curl -sLf "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" > /usr/local/bin/kubectl \ 29 | && chmod +x /usr/local/bin/kubectl \ 30 | && apk del curl 31 | 32 | RUN mkdir -p /opt/cni/bin 33 | 34 | COPY --from=cni_base /opt/cni/bin /tmp 35 | 36 | ENV ETCD_URL=https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-${ARCH}.tar.gz 37 | 38 | RUN wget -q -O - "${ETCD_URL}" | tar xzf - -C /tmp && \ 39 | mv /tmp/etcd-*/etcdctl /usr/local/bin/etcdctl && \ 40 | rm -rf /tmp/etcd-* && rm -f /etcd-*.tar.gz && \ 41 | apk del wget 42 | 43 | COPY templates /etc/confd/templates/ 44 | COPY conf.d /etc/confd/conf.d/ 45 | COPY cert-deployer nginx-proxy /usr/bin/ 46 | COPY entrypoint.sh cloud-provider.sh weave-plugins-cni.sh /opt/rke-tools/ 47 | COPY rke-etcd-backup /opt/rke-tools 48 | 49 | VOLUME /opt/rke-tools 50 | CMD ["/bin/bash"] 51 | 52 | # Temporary image mostly to verify all binaries exist and are 53 | # valid for the target architecture. 54 | FROM tonistiigi/xx:1.4.0 AS xx 55 | FROM base as test 56 | COPY --from=xx / / 57 | 58 | ARG TARGETOS=linux 59 | ARG TARGETARCH=${ARCH} 60 | 61 | RUN xx-verify --static /tmp/bandwidth \ 62 | && xx-verify --static /tmp/bridge \ 63 | && xx-verify --static /tmp/dhcp \ 64 | && xx-verify --static /tmp/firewall \ 65 | && xx-verify --static /tmp/flannel \ 66 | && xx-verify --static /tmp/host-device \ 67 | && xx-verify --static /tmp/host-local \ 68 | && xx-verify --static /tmp/ipvlan \ 69 | && xx-verify --static /tmp/loopback \ 70 | && xx-verify --static /tmp/macvlan \ 71 | && xx-verify --static /tmp/portmap \ 72 | && xx-verify --static /tmp/ptp \ 73 | && xx-verify --static /tmp/sbr \ 74 | && xx-verify --static /tmp/static \ 75 | && xx-verify --static /tmp/tuning \ 76 | && xx-verify --static /tmp/vlan \ 77 | && xx-verify --static /tmp/vrf 78 | 79 | RUN xx-verify --static /opt/rke-tools/bin/cri-dockerd \ 80 | && xx-verify --static /opt/rke-tools/bin/docker \ 81 | && xx-verify --static /opt/rke-tools/rke-etcd-backup 82 | 83 | RUN xx-verify --static /usr/bin/confd \ 84 | && xx-verify --static /usr/local/bin/kubectl 85 | 86 | FROM base as final 87 | -------------------------------------------------------------------------------- /docs/components.md: -------------------------------------------------------------------------------- 1 | # Components 2 | 3 | ## Entrypoint for each Kubernetes container 4 | 5 | Each node in an RKE cluster gets a container named `service-sidekick` created. It will stay in `Created` state, it will not show up in `docker ps` output (only in `docker ps -a`). The purpose of this container is to share its volume to RKE Kubernetes containers. Each RKE Kubernetes container is started with `--volumes from=service-sidekick` so each container can use files from that volume. The volume defined in the `Dockerfile` (`package/Dockerfile`) is `/opt/rke-tools`. The default entrypoint for RKE Kubernetes containers is `/opt/rke-tools/entrypoint.sh` (https://github.com/rancher/rke/blob/v1.4.8/cluster/plan.go#L46). 6 | 7 | The comments in `entrypoint.sh` should explain what is being done. This also includes the use of `cloud-provider.sh`. 8 | 9 | ## etcd snapshots 10 | 11 | The compiled binary from `main.go` is `rke-etcd-backup`. This binary is used to operate etcd snapshots, used by the RKE etcd containers on nodes with the etcd role. 12 | 13 | The binary has one subcommand (`etcd-backup`) with multiple options, which are described below. 14 | 15 | ### save 16 | 17 | Used in container to create snapshots in interval (`etcd-rolling-snapshots`) or during ad-hoc snapshots (`etcd-snapshot-once`) using the `--once` flag. 18 | 19 | ### delete 20 | 21 | Used to delete created snapshots locally or uploaded to S3 22 | 23 | ### download 24 | 25 | Used to download snapshots from S3 or download snapshots from other etcd nodes. Each node takes its own snapshot but only one node's snapshot is selected for restore. The selected node's snapshot is served in a container for the remaining etcd nodes to download, to make sure they are all using the exact same snapshot source. 26 | 27 | ### serve 28 | 29 | Used to serve the selected snapshot for restore to the other etcd nodes. This will create an HTTPS endpoint for the other nodes to download the snapshot archive that can be used for the restore. 30 | 31 | ### extractstatefile 32 | 33 | Used to extract the RKE statefile from an etcd snapshot archive. Starting with RKE v1.1.4, the statefile got included in the snapshot archive to make sure the correct information was available to restore (like Kubernetes certificates, reference: https://github.com/rancher/rke/issues/1336). This is used when a restore is requested and the statefile is needed. 34 | 35 | ## Container to run the proxy between `kubelet` and `kube-apiserver` in RKE clusters 36 | 37 | The kubelet connects to the `kube-apiserver` using a container named `nginx-proxy`. The `nginx-proxy` container runs on the host network and nginx listens on port 6443. The `nginx-proxy` container is configured with the environment variable `CP_HOSTS` which contains the IP addresses of all controlplane nodes in the cluster. The container will use `confd` to dynamically generate the `/etc/nginx/nginx.conf` before starting nginx itself. The file used by confd can be found in `conf.d/nginx.toml`, the template used by confd can be found in `templates/nginx.tmpl`. 38 | 39 | ## Container to deploy Kubernetes certificates needed by the nodes in RKE clusters 40 | 41 | `cert-deployer` is a bash script helper to deploy certificates to RKE Kubernetes nodes using environment variables. RKE will set the environment variables and create the container with `cert-deployer` as command. The container will create the certificate files in the correct location. 42 | 43 | ## Container to deploy portmap CNI plugin for Weave 44 | 45 | Weave needs the `portmap` CNI plugin starting with v2.5.0, `weave-plugins-cni.sh` is a bash script helper to deploy this CNI plugin 46 | 47 | ## Container to make mounts shared (DEPRECATED) 48 | 49 | `share-mnt` was created to make mounts shared, mostly used to make RKE/Rancher compatible with running on boot2docker. Error shown was `[workerPlane] Failed to bring up Worker Plane: Failed to start [kubelet] container on host [192.168.59.104]: Error response from daemon: linux mounts: Path /var/lib/kubelet is mounted on / but it is not a shared mount`. See https://github.com/rancher/rancher/issues/12687 for more details. This is no longer in use. 50 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 2 | github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= 3 | github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 8 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 9 | github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= 10 | github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 11 | github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= 12 | github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= 13 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 14 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 15 | github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= 16 | github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 17 | github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 18 | github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= 19 | github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= 20 | github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= 21 | github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= 22 | github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0= 23 | github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= 24 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 25 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 26 | github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= 27 | github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 28 | github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= 29 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 30 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 31 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 32 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 33 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 34 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 35 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 36 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 37 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 38 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 39 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 40 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 41 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 42 | github.com/urfave/cli v1.22.15 h1:nuqt+pdC/KqswQKhETJjo7pvn/k4xMUxgW6liI7XpnM= 43 | github.com/urfave/cli v1.22.15/go.mod h1:wSan1hmo5zeyLGBjRJbzRTNk8gwoYa2B9n4q9dmRIc0= 44 | golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= 45 | golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= 46 | golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= 47 | golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= 48 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 49 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 50 | golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= 51 | golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 52 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 53 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 54 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 55 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 56 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 57 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 58 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 59 | -------------------------------------------------------------------------------- /.github/workflows/workflow.yaml: -------------------------------------------------------------------------------- 1 | name: Workflow for rke-tools 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | tags: 8 | - '*' 9 | pull_request: 10 | branches: 11 | - master 12 | 13 | env: 14 | IMAGE: rancher/rke-tools 15 | 16 | jobs: 17 | ci: 18 | permissions: 19 | contents: read 20 | runs-on: ubuntu-latest 21 | timeout-minutes: 10 22 | steps: 23 | - name: Checkout code 24 | uses: actions/checkout@v4 25 | - name: Setup Go 26 | uses: actions/setup-go@v5 27 | with: 28 | go-version-file: 'go.mod' 29 | - name: Run golangci-lint 30 | uses: golangci/golangci-lint-action@v6 31 | with: 32 | version: v1.58 33 | args: --disable-all -E revive 34 | - name: Build 35 | run: | 36 | ./scripts/ci 37 | 38 | build-and-push: 39 | permissions: 40 | contents: read 41 | id-token: write 42 | runs-on: ubuntu-latest 43 | timeout-minutes: 30 44 | needs: ci 45 | if: github.event_name == 'push' && github.ref_type == 'tag' 46 | strategy: 47 | fail-fast: true 48 | matrix: 49 | arch: [amd64, arm64] 50 | steps: 51 | - name: Checkout code 52 | uses: actions/checkout@v4 53 | - name: Setup Go 54 | uses: actions/setup-go@v5 55 | - name: Build 56 | run: | 57 | ./scripts/build 58 | env: 59 | GOARCH: ${{ matrix.arch }} 60 | - name: Fix permissions 61 | run: | 62 | mv bin/rke-etcd-backup . 63 | chmod 755 rke-etcd-backup 64 | - name: Docker meta 65 | id: meta 66 | uses: docker/metadata-action@v5 67 | with: 68 | images: ${{ env.IMAGE }} 69 | flavor: | 70 | latest=false 71 | - name: Set up QEMU 72 | uses: docker/setup-qemu-action@v3 73 | - name: Set up Docker Buildx 74 | uses: docker/setup-buildx-action@v3 75 | - name: Load Secrets from Vault 76 | uses: rancher-eio/read-vault-secrets@main 77 | with: 78 | secrets: | 79 | secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; 80 | secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD 81 | - name: Login to Docker Hub 82 | uses: docker/login-action@v3 83 | with: 84 | username: ${{ env.DOCKER_USERNAME }} 85 | password: ${{ env.DOCKER_PASSWORD }} 86 | - name: Build and push Docker image 87 | id: build 88 | uses: docker/build-push-action@v6 89 | with: 90 | context: . 91 | file: package/Dockerfile 92 | push: true 93 | platforms: linux/${{ matrix.arch }} 94 | tags: ${{ env.IMAGE }}:${{ github.ref_name }}-linux-${{ matrix.arch }} 95 | build-args: ARCH=${{ matrix.arch }} 96 | labels: "${{ steps.meta.outputs.labels }}" 97 | - name: Export digest 98 | run: | 99 | mkdir -p /tmp/digests 100 | digest="${{ steps.build.outputs.digest }}" 101 | touch "/tmp/digests/${digest#sha256:}" 102 | - name: Upload digest 103 | uses: actions/upload-artifact@v4 104 | with: 105 | name: "digests-linux-${{ matrix.arch }}" 106 | path: /tmp/digests/* 107 | if-no-files-found: error 108 | retention-days: 1 109 | overwrite: true 110 | 111 | merge: 112 | runs-on: ubuntu-latest 113 | needs: build-and-push 114 | permissions: 115 | contents: read 116 | id-token: write 117 | timeout-minutes: 10 118 | if: github.event_name == 'push' && github.ref_type == 'tag' 119 | steps: 120 | - name: Download digests 121 | uses: actions/download-artifact@v4 122 | with: 123 | path: /tmp/digests 124 | pattern: digests-* 125 | merge-multiple: true 126 | - name: Set up Docker Buildx 127 | uses: docker/setup-buildx-action@v3 128 | - name: Docker meta 129 | id: meta 130 | uses: docker/metadata-action@v5 131 | with: 132 | images: ${{ env.IMAGE }} 133 | flavor: | 134 | latest=false 135 | - name: Load Secrets from Vault 136 | uses: rancher-eio/read-vault-secrets@main 137 | with: 138 | secrets: | 139 | secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; 140 | secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD 141 | - name: Login to Docker Hub 142 | uses: docker/login-action@v3 143 | with: 144 | username: ${{ env.DOCKER_USERNAME }} 145 | password: ${{ env.DOCKER_PASSWORD }} 146 | - name: Create manifest list and push 147 | working-directory: /tmp/digests 148 | run: | 149 | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ 150 | $(printf '${{ env.IMAGE }}@sha256:%s ' *) 151 | - name: Inspect image 152 | run: | 153 | docker buildx imagetools inspect ${{ env.IMAGE }}:${{ steps.meta.outputs.version }} 154 | -------------------------------------------------------------------------------- /cloud-provider.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | AZURE_META_URL="http://169.254.169.254/metadata/instance/compute" 4 | AZURE_META_API_VERSION="2019-08-15" 5 | AZURE_CLOUD_CONFIG_PATH="/etc/kubernetes/cloud-config" 6 | 7 | set_azure_config() { 8 | set +x 9 | local az_resources_group=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .resourceGroup) 10 | local az_subscription_id=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .subscriptionId) 11 | local az_location=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .location) 12 | local azure_cloud=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .cloud) 13 | local azure_client_id=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .aadClientId) 14 | local azure_client_secret=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .aadClientSecret) 15 | local azure_tenant_id=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .tenantId) 16 | local az_vm_nsg=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .securityGroupName) 17 | local az_vnet_resource_group=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .vnetResourceGroup) 18 | local az_subnet_name=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .subnetName) 19 | local az_vnet_name=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .vnetName) 20 | local az_vm_type=$(cat "$AZURE_CLOUD_CONFIG_PATH" | jq -r .vmType) 21 | 22 | local az_vm_resources_group=$(curl -s -H Metadata:true "${AZURE_META_URL}/resourceGroupName?api-version=${AZURE_META_API_VERSION}&format=text") 23 | local az_vm_name=$(curl -s -H Metadata:true "${AZURE_META_URL}/name?api-version=${AZURE_META_API_VERSION}&format=text") 24 | 25 | # setting correct login cloud 26 | if [ "${azure_cloud}" = "null" ] || [ "${azure_cloud}" = "" ]; then 27 | azure_cloud="AzureCloud" 28 | fi 29 | if [ "${azure_cloud}" = "AzureUSGovernmentCloud" ]; then 30 | azure_cloud="AzureUSGovernment" # naming issue with azure cli 31 | fi 32 | az cloud set --name ${azure_cloud} 33 | 34 | # login to Azure 35 | az login --service-principal -u ${azure_client_id} -p ${azure_client_secret} --tenant ${azure_tenant_id} 2>&1 > /dev/null 36 | 37 | # set subscription to be the current active subscription 38 | az account set --subscription ${az_subscription_id} 39 | 40 | if [ -z "$az_resources_group" ] ; then 41 | az_resources_group="$az_vm_resources_group" 42 | fi 43 | 44 | if [ -z "$az_location" ]; then 45 | az_location=$(curl -s -H Metadata:true "${AZURE_META_URL}/location?api-version=${AZURE_META_API_VERSION}&format=text") 46 | fi 47 | 48 | if [ "$az_vm_type" = "vmss" ]; then 49 | # vmss 50 | local az_vm_scale_set_name=$(curl -s -H Metadata:true "${AZURE_META_URL}/vmScaleSetName?api-version=${AZURE_META_API_VERSION}&format=text") 51 | local az_vm_instance_id=$(az vmss list-instances -g ${az_resources_group} --name ${az_vm_scale_set_name} --query "[?name=='${az_vm_name}'].instanceId" --output tsv) 52 | local az_vm_nic=$(az vmss nic list -g ${az_resources_group} --vmss-name ${az_vm_scale_set_name} --output tsv --query [0].name) 53 | 54 | if [ -z "$az_subnet_name" ] ; then 55 | az_subnet_name=$(az vmss nic show -g ${az_resources_group} --vmss-name ${az_vm_scale_set_name} --name ${az_vm_nic} --instance-id ${az_vm_instance_id} | jq -r .ipConfigurations[0].subnet.id | cut -d "/" -f 11) 56 | fi 57 | 58 | if [ -z "$az_vnet_name" ] ; then 59 | az_vnet_name=$(az vmss nic show -g ${az_resources_group} --vmss-name ${az_vm_scale_set_name} --name ${az_vm_nic} --instance-id ${az_vm_instance_id} | jq -r .ipConfigurations[0].subnet.id | cut -d "/" -f 9) 60 | fi 61 | 62 | if [ -z "$az_vnet_resource_group" ] ; then 63 | az_vnet_resource_group=$(az vmss nic show -g ${az_resources_group} --vmss-name ${az_vm_scale_set_name} --name ${az_vm_nic} --instance-id ${az_vm_instance_id} | jq -r .ipConfigurations[0].subnet.id | cut -d "/" -f 5) 64 | fi 65 | 66 | if [ -z "$az_vm_nsg" ] ; then 67 | az_vm_nsg=$(az vmss nic show -g ${az_resources_group} --vmss-name ${az_vm_scale_set_name} --name ${az_vm_nic} --instance-id ${az_vm_instance_id} | jq -r .networkSecurityGroup.id | cut -d "/" -f 9) 68 | fi 69 | else 70 | # standard, vm 71 | local az_vm_nic=$(az vm nic list -g ${az_resources_group} --vm-name ${az_vm_name} | jq -r .[0].id | cut -d "/" -f 9) 72 | 73 | if [ -z "$az_subnet_name" ] ; then 74 | az_subnet_name=$(az vm nic show -g ${az_resources_group} --vm-name ${az_vm_name} --nic ${az_vm_nic} | jq -r .ipConfigurations[0].subnet.id | cut -d "/" -f 11) 75 | fi 76 | 77 | if [ -z "$az_vnet_name" ] ; then 78 | az_vnet_name=$(az vm nic show -g ${az_resources_group} --vm-name ${az_vm_name} --nic ${az_vm_nic} | jq -r .ipConfigurations[0].subnet.id | cut -d "/" -f 9) 79 | fi 80 | 81 | if [ -z "$az_vnet_resource_group" ] ; then 82 | az_vnet_resource_group=$(az vm nic show -g ${az_resources_group} --vm-name ${az_vm_name} --nic ${az_vm_nic} | jq -r .ipConfigurations[0].subnet.id | cut -d "/" -f 5) 83 | fi 84 | 85 | if [ -z "$az_vm_nsg" ] ; then 86 | az_vm_nsg=$(az vm nic show -g ${az_resources_group} --vm-name ${az_vm_name} --nic ${az_vm_nic} | jq -r .networkSecurityGroup.id | cut -d "/" -f 9) 87 | fi 88 | fi 89 | 90 | az logout 2>&1 > /dev/null 91 | 92 | if [ -z "$az_subscription_id" ] || [ -z "$az_location" ] || [ -z "$az_resources_group" ] || [ -z "$az_vnet_resource_group" ] || [ -z "$az_subnet_name" ] || [ -z "$az_vnet_name" ] || [ -z "$az_vm_nsg" ]; then 93 | echo "Some variables were not populated correctly, using the passed config!" 94 | else 95 | local cloud_config_temp=$(mktemp) 96 | cat "$AZURE_CLOUD_CONFIG_PATH" |\ 97 | jq '.subscriptionId=''"'${az_subscription_id}'"''' |\ 98 | jq '.location=''"'${az_location}'"''' |\ 99 | jq '.resourceGroup=''"'${az_resources_group}'"''' |\ 100 | jq '.vnetResourceGroup=''"'${az_vnet_resource_group}'"''' |\ 101 | jq '.subnetName=''"'${az_subnet_name}'"''' |\ 102 | jq '.useInstanceMetadata=true' |\ 103 | jq '.securityGroupName=''"'${az_vm_nsg}'"''' |\ 104 | jq '.vnetName=''"'${az_vnet_name}'"''' > $cloud_config_temp 105 | # move the temp to the azure cloud config path 106 | mv $cloud_config_temp $AZURE_CLOUD_CONFIG_PATH 107 | fi 108 | } 109 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | # Evaluate the iptables mode every time the container starts (fixes OS upgrades changing mode) 6 | if [ "$1" = "kubelet" ] || [ "$1" = "kube-proxy" ]; then 7 | update-alternatives --set iptables /usr/sbin/iptables-wrapper 8 | fi 9 | 10 | # br_netfilter is required for canal and flannel network plugins. 11 | if [ "$1" = "kube-proxy" ] && [ "${RKE_KUBE_PROXY_BR_NETFILTER}" = "true" ]; then 12 | modprobe br_netfilter || true 13 | fi 14 | 15 | # generate Azure cloud provider config if configured 16 | if echo ${@} | grep -q "cloud-provider=azure"; then 17 | if [ "$1" = "kubelet" ] || [ "$1" = "kube-apiserver" ] || [ "$1" = "kube-controller-manager" ]; then 18 | source /opt/rke-tools/cloud-provider.sh 19 | set_azure_config 20 | # If set_azure_config is called debug needs to be turned back on 21 | set -x 22 | fi 23 | fi 24 | 25 | # In case of AWS cloud provider being configured, RKE will not set `hostname-override` flag because it needs to match the node/instance name in AWS. 26 | # This will query EC2 metadata and use the value for setting `hostname-override` to match the node/instance name. 27 | # RKE pull request: https://github.com/rancher/rke/pull/2803 28 | if [ "$1" = "kube-proxy" ] || [ "$1" = "kubelet" ]; then 29 | if echo ${@} | grep -v "hostname-override"; then 30 | aws_api_token=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60") 31 | hostname=$(curl -H "X-aws-ec2-metadata-token: $aws_api_token" "http://169.254.169.254/latest/meta-data/hostname") 32 | if [ -z "$hostname" ]; then 33 | hostname=$(hostname -f) 34 | fi 35 | set ${@} --hostname-override=$hostname 36 | fi 37 | fi 38 | 39 | # Prepare kubelet for running inside container 40 | if [ "$1" = "kubelet" ]; then 41 | CGROUPDRIVER=$(/opt/rke-tools/bin/docker info -f '{{.Info.CgroupDriver}}') 42 | CGROUPVERSION=$(/opt/rke-tools/bin/docker info -f '{{.Info.CgroupVersion}}') 43 | DOCKER_ROOT=$(DOCKER_API_VERSION=1.24 /opt/rke-tools/bin/docker info -f '{{.Info.DockerRootDir}}') 44 | DOCKER_DIRS=$(find -O1 $DOCKER_ROOT -maxdepth 1) # used to exclude mounts that are subdirectories of $DOCKER_ROOT to ensure we don't unmount mounted filesystems on sub directories 45 | for i in $DOCKER_ROOT /var/lib/docker /run /var/run; do 46 | for m in $(tac /proc/mounts | awk '{print $2}' | grep ^${i}/); do 47 | if [ "$m" != "/var/run/nscd" ] && [ "$m" != "/run/nscd" ] && ! echo $DOCKER_DIRS | grep -qF "$m"; then 48 | umount $m || true 49 | fi 50 | done 51 | done 52 | mount --rbind /host/dev /dev 53 | mount -o rw,remount /sys/fs/cgroup 2>/dev/null || true 54 | 55 | # Only applicable to cgroup v1 56 | if [ "${CGROUPVERSION}" -eq 1 ]; then 57 | for i in /sys/fs/cgroup/*; do 58 | if [ -d $i ]; then 59 | mkdir -p $i/kubepods 60 | fi 61 | done 62 | 63 | mkdir -p /sys/fs/cgroup/cpuacct,cpu/ 64 | mount --bind /sys/fs/cgroup/cpu,cpuacct/ /sys/fs/cgroup/cpuacct,cpu/ 65 | mkdir -p /sys/fs/cgroup/net_prio,net_cls/ 66 | mount --bind /sys/fs/cgroup/net_cls,net_prio/ /sys/fs/cgroup/net_prio,net_cls/ 67 | fi 68 | 69 | # If we are running on SElinux host, need to: 70 | mkdir -p /opt/cni /etc/cni 71 | chcon -Rt svirt_sandbox_file_t /etc/cni 2>/dev/null || true 72 | chcon -Rt svirt_sandbox_file_t /opt/cni 2>/dev/null || true 73 | 74 | # Set this to 1 as required by network plugins 75 | # https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#network-plugin-requirements 76 | sysctl -w net.bridge.bridge-nf-call-iptables=1 || true 77 | 78 | # Mount host os-release so kubelet can report the correct OS 79 | if [ -f /host/usr/lib/os-release ]; then 80 | ln -sf /host/usr/lib/os-release /usr/lib/os-release 81 | elif [ -f /host/etc/os-release ]; then 82 | ln -sf /host/etc/os-release /usr/lib/os-release 83 | elif [ -f /host/usr/share/ros/os-release ]; then 84 | ln -sf /host/usr/share/ros/os-release /usr/lib/os-release 85 | fi 86 | 87 | # Check if no other or additional resolv-conf is passed (default is configured as /etc/resolv.conf) 88 | if echo "$@" | grep -q -- --resolv-conf=/etc/resolv.conf; then 89 | # Check if host is running `system-resolved` 90 | if pgrep -f systemd-resolved > /dev/null; then 91 | # Check if the resolv.conf with the actual nameservers is present 92 | if [ -f /run/systemd/resolve/resolv.conf ]; then 93 | RESOLVCONF="--resolv-conf=/run/systemd/resolve/resolv.conf" 94 | fi 95 | fi 96 | fi 97 | 98 | if [ ! -z "${RKE_KUBELET_DOCKER_CONFIG}" ] 99 | then 100 | echo ${RKE_KUBELET_DOCKER_CONFIG} | base64 -d | tee ${RKE_KUBELET_DOCKER_FILE} 101 | fi 102 | 103 | # separate flow for cri-dockerd to minimize change to the existing way we run kubelet 104 | if [ "${RKE_KUBELET_CRIDOCKERD}" == "true" ]; then 105 | 106 | # Mount kubelet docker config to /.docker/config.json 107 | if [ ! -z "${RKE_KUBELET_DOCKER_CONFIG}" ] 108 | then 109 | mkdir -p /.docker && touch /.docker/config.json 110 | mount --bind ${RKE_KUBELET_DOCKER_FILE} /.docker/config.json 111 | fi 112 | 113 | # Get the value of pause image to start cri-dockerd 114 | RKE_KUBELET_PAUSEIMAGE=$(echo "$@" | grep -Eo "\-\-pod-infra-container-image+.*" | awk '{print $1}') 115 | CONTAINER_RUNTIME_ENDPOINT=$(echo "$@" | grep -Eo "\-\-container-runtime-endpoint+.*" | awk '{print $1}' | cut -d "=" -f2) 116 | if [ "$CONTAINER_RUNTIME_ENDPOINT" == "/var/run/dockershim.sock" ]; then 117 | # cri-dockerd v0.3.11 requires unix socket or tcp endpoint, update old endpoint passed by rke 118 | CONTAINER_RUNTIME_ENDPOINT="unix://$CONTAINER_RUNTIME_ENDPOINT" 119 | fi 120 | EXTRA_FLAGS="" 121 | if [ "${RKE_KUBELET_CRIDOCKERD_DUALSTACK}" == "true" ]; then 122 | EXTRA_FLAGS="--ipv6-dual-stack" 123 | fi 124 | if [ -z "${CRIDOCKERD_STREAM_SERVER_ADDRESS}" ]; then 125 | CRIDOCKERD_STREAM_SERVER_ADDRESS="127.0.0.1" 126 | fi 127 | if [ -z "${CRIDOCKERD_STREAM_SERVER_PORT}" ]; then 128 | CRIDOCKERD_STREAM_SERVER_PORT="10010" 129 | fi 130 | 131 | /opt/rke-tools/bin/cri-dockerd --network-plugin="cni" --cni-conf-dir="/etc/cni/net.d" --cni-bin-dir="/opt/cni/bin" ${RKE_KUBELET_PAUSEIMAGE} --container-runtime-endpoint=$CONTAINER_RUNTIME_ENDPOINT --streaming-bind-addr=${CRIDOCKERD_STREAM_SERVER_ADDRESS}:${CRIDOCKERD_STREAM_SERVER_PORT} ${EXTRA_FLAGS} & 132 | 133 | # wait for cri-dockerd to start as kubelet depends on it 134 | echo "Sleeping 10 waiting for cri-dockerd to start" 135 | sleep 10 136 | 137 | # start kubelet 138 | exec "$@" --cgroup-driver=$CGROUPDRIVER $RESOLVCONF & 139 | 140 | # waiting for either cri-dockerd or kubelet to crash and exit so it can be restarted 141 | wait -n 142 | exit $? 143 | else 144 | # start kubelet 145 | exec "$@" --cgroup-driver=$CGROUPDRIVER $RESOLVCONF 146 | fi 147 | fi 148 | 149 | exec "$@" 150 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "context" 7 | "crypto/tls" 8 | "crypto/x509" 9 | "encoding/base64" 10 | "encoding/json" 11 | "encoding/pem" 12 | "fmt" 13 | "io" 14 | "net/http" 15 | "net/url" 16 | "os" 17 | "os/exec" 18 | "path" 19 | "path/filepath" 20 | "regexp" 21 | "strings" 22 | "time" 23 | 24 | "github.com/minio/minio-go/v7" 25 | "github.com/minio/minio-go/v7/pkg/credentials" 26 | log "github.com/sirupsen/logrus" 27 | "github.com/urfave/cli" 28 | ) 29 | 30 | const ( 31 | backupBaseDir = "/backup" 32 | defaultBackupRetries = 4 33 | clusterStateExtension = "rkestate" 34 | compressedExtension = "zip" 35 | contentType = "application/zip" 36 | k8sBaseDir = "/etc/kubernetes" 37 | defaultS3Retries = 3 38 | serverPort = "2379" 39 | s3Endpoint = "s3.amazonaws.com" 40 | tmpStateFilePath = "/tmp/cluster.rkestate" 41 | failureInterval = 15 * time.Second 42 | ) 43 | 44 | var ( 45 | backupRetries uint = defaultBackupRetries 46 | s3Retries uint = defaultS3Retries 47 | ) 48 | 49 | var commonFlags = []cli.Flag{ 50 | cli.StringFlag{ 51 | Name: "endpoints", 52 | Usage: "Etcd endpoints", 53 | Value: "127.0.0.1:2379", 54 | }, 55 | cli.BoolFlag{ 56 | Name: "debug", 57 | Usage: "Verbose logging information for debugging purposes", 58 | EnvVar: "RANCHER_DEBUG", 59 | }, 60 | cli.StringFlag{ 61 | Name: "name", 62 | Usage: "Backup name to take once", 63 | }, 64 | cli.StringFlag{ 65 | Name: "cacert", 66 | Usage: "Etcd CA client certificate path", 67 | EnvVar: "ETCD_CACERT", 68 | }, 69 | cli.StringFlag{ 70 | Name: "cert", 71 | Usage: "Etcd client certificate path", 72 | EnvVar: "ETCD_CERT", 73 | }, 74 | cli.StringFlag{ 75 | Name: "key", 76 | Usage: "Etcd client key path", 77 | EnvVar: "ETCD_KEY", 78 | }, 79 | cli.StringFlag{ 80 | Name: "local-endpoint", 81 | Usage: "Local backup download endpoint", 82 | EnvVar: "LOCAL_ENDPOINT", 83 | }, 84 | cli.BoolFlag{ 85 | Name: "s3-backup", 86 | Usage: "Backup etcd snapshot to your s3 server, set true or false", 87 | EnvVar: "S3_BACKUP", 88 | }, 89 | cli.StringFlag{ 90 | Name: "s3-endpoint", 91 | Usage: "Specify s3 endpoint address", 92 | EnvVar: "S3_ENDPOINT", 93 | }, 94 | cli.StringFlag{ 95 | Name: "s3-accessKey", 96 | Usage: "Specify s3 access key", 97 | EnvVar: "S3_ACCESS_KEY", 98 | }, 99 | cli.StringFlag{ 100 | Name: "s3-secretKey", 101 | Usage: "Specify s3 secret key", 102 | EnvVar: "S3_SECRET_KEY", 103 | }, 104 | cli.StringFlag{ 105 | Name: "s3-bucketName", 106 | Usage: "Specify s3 bucket name", 107 | EnvVar: "S3_BUCKET_NAME", 108 | }, 109 | cli.StringFlag{ 110 | Name: "s3-region", 111 | Usage: "Specify s3 bucket region", 112 | EnvVar: "S3_BUCKET_REGION", 113 | }, 114 | cli.StringFlag{ 115 | Name: "s3-endpoint-ca", 116 | Usage: "Specify custom CA for S3 endpoint. Can be a file path or a base64 string", 117 | EnvVar: "S3_ENDPOINT_CA", 118 | }, 119 | cli.StringFlag{ 120 | Name: "s3-folder", 121 | Usage: "Specify folder for snapshots", 122 | EnvVar: "S3_FOLDER", 123 | }, 124 | } 125 | 126 | var deleteFlags = []cli.Flag{ 127 | cli.StringFlag{ 128 | Name: "name", 129 | Usage: "snapshot name to delete", 130 | }, 131 | cli.BoolFlag{ 132 | Name: "s3-backup", 133 | Usage: "delete snapshot from s3", 134 | }, 135 | cli.BoolFlag{ 136 | Name: "cleanup", 137 | Usage: "delete uncompressed files only", 138 | }, 139 | cli.StringFlag{ 140 | Name: "s3-endpoint", 141 | Usage: "Specify s3 endpoint address", 142 | EnvVar: "S3_ENDPOINT", 143 | }, 144 | cli.StringFlag{ 145 | Name: "s3-accessKey", 146 | Usage: "Specify s3 access key", 147 | EnvVar: "S3_ACCESS_KEY", 148 | }, 149 | cli.StringFlag{ 150 | Name: "s3-secretKey", 151 | Usage: "Specify s3 secret key", 152 | EnvVar: "S3_SECRET_KEY", 153 | }, 154 | cli.StringFlag{ 155 | Name: "s3-bucketName", 156 | Usage: "Specify s3 bucket name", 157 | EnvVar: "S3_BUCKET_NAME", 158 | }, 159 | cli.StringFlag{ 160 | Name: "s3-region", 161 | Usage: "Specify s3 bucket region", 162 | EnvVar: "S3_BUCKET_REGION", 163 | }, 164 | cli.StringFlag{ 165 | Name: "s3-endpoint-ca", 166 | Usage: "Specify custom CA for S3 endpoint. Can be a file path or a base64 string", 167 | EnvVar: "S3_ENDPOINT_CA", 168 | }, 169 | cli.StringFlag{ 170 | Name: "s3-folder", 171 | Usage: "Specify folder for snapshots", 172 | EnvVar: "S3_FOLDER", 173 | }, 174 | } 175 | 176 | type backupConfig struct { 177 | Backup bool 178 | Endpoint string 179 | AccessKey string 180 | SecretKey string 181 | BucketName string 182 | Region string 183 | EndpointCA string 184 | Folder string 185 | } 186 | 187 | func init() { 188 | log.SetOutput(os.Stderr) 189 | } 190 | 191 | func main() { 192 | err := os.Setenv("ETCDCTL_API", "3") 193 | if err != nil { 194 | log.Fatal(err) 195 | } 196 | 197 | app := cli.NewApp() 198 | app.Name = "Etcd Wrapper" 199 | app.Usage = "Utility services for Etcd cluster backup" 200 | app.Commands = []cli.Command{ 201 | BackupCommand(), 202 | } 203 | err = app.Run(os.Args) 204 | if err != nil { 205 | log.Fatal(err) 206 | } 207 | } 208 | 209 | func BackupCommand() cli.Command { 210 | 211 | snapshotFlags := []cli.Flag{ 212 | cli.DurationFlag{ 213 | Name: "creation", 214 | Usage: "Create backups after this time interval in minutes", 215 | Value: 5 * time.Minute, 216 | }, 217 | cli.DurationFlag{ 218 | Name: "retention", 219 | Usage: "Retain backups within this time interval in hours", 220 | Value: 24 * time.Hour, 221 | }, 222 | cli.BoolFlag{ 223 | Name: "once", 224 | Usage: "Take backup only once", 225 | }, 226 | } 227 | 228 | snapshotFlags = append(snapshotFlags, commonFlags...) 229 | 230 | return cli.Command{ 231 | Name: "etcd-backup", 232 | Usage: "Perform etcd backup tools", 233 | Subcommands: []cli.Command{ 234 | { 235 | Name: "save", 236 | Usage: "Take snapshot on all etcd hosts and backup to s3 compatible storage", 237 | Flags: append(snapshotFlags, cli.UintFlag{ 238 | Name: "backup-retries", 239 | Usage: "Number of times to attempt the backup", 240 | Destination: &backupRetries, 241 | }, cli.UintFlag{ 242 | Name: "s3-retries", 243 | Usage: "Number of times to attempt the upload to s3", 244 | Destination: &s3Retries, 245 | }), 246 | Action: SaveBackupAction, 247 | }, 248 | { 249 | Name: "delete", 250 | Usage: "Delete snapshot from etcd hosts or s3 compatible storage", 251 | Flags: deleteFlags, 252 | Action: DeleteBackupAction, 253 | }, 254 | { 255 | Name: "download", 256 | Usage: "Download specified snapshot from s3 compatible storage or another local endpoint", 257 | Flags: commonFlags, 258 | Action: DownloadBackupAction, 259 | }, 260 | { 261 | Name: "extractstatefile", 262 | Usage: "Extract statefile for specified snapshot (if it is included in the archive)", 263 | Flags: snapshotFlags, 264 | Action: ExtractStateFileAction, 265 | }, 266 | { 267 | Name: "serve", 268 | Usage: "Provide HTTPS endpoint to pull local snapshot", 269 | Flags: []cli.Flag{ 270 | cli.StringFlag{ 271 | Name: "name", 272 | Usage: "Backup name to take once", 273 | }, 274 | cli.StringFlag{ 275 | Name: "cacert", 276 | Usage: "Etcd CA client certificate path", 277 | EnvVar: "ETCD_CACERT", 278 | }, 279 | cli.StringFlag{ 280 | Name: "cert", 281 | Usage: "Etcd client certificate path", 282 | EnvVar: "ETCD_CERT", 283 | }, 284 | cli.StringFlag{ 285 | Name: "key", 286 | Usage: "Etcd client key path", 287 | EnvVar: "ETCD_KEY", 288 | }, 289 | }, 290 | Action: ServeBackupAction, 291 | }, 292 | }, 293 | } 294 | } 295 | 296 | func SetLoggingLevel(debug bool) { 297 | if debug { 298 | log.SetLevel(log.DebugLevel) 299 | log.Debug("Log level set to debug") 300 | } else { 301 | log.SetLevel(log.InfoLevel) 302 | } 303 | } 304 | 305 | func SaveBackupAction(c *cli.Context) error { 306 | SetLoggingLevel(c.Bool("debug")) 307 | 308 | creationPeriod := c.Duration("creation") 309 | retentionPeriod := c.Duration("retention") 310 | etcdCert := c.String("cert") 311 | etcdCACert := c.String("cacert") 312 | etcdKey := c.String("key") 313 | etcdEndpoints := c.String("endpoints") 314 | if creationPeriod == 0 || retentionPeriod == 0 { 315 | log.WithFields(log.Fields{ 316 | "creation": creationPeriod, 317 | "retention": retentionPeriod, 318 | }).Errorf("Creation period and/or retention are not set") 319 | return fmt.Errorf("Creation period and/or retention are not set") 320 | } 321 | 322 | if len(etcdCert) == 0 || len(etcdCACert) == 0 || len(etcdKey) == 0 { 323 | log.WithFields(log.Fields{ 324 | "etcdCert": etcdCert, 325 | "etcdCACert": etcdCACert, 326 | "etcdKey": etcdKey, 327 | }).Errorf("Failed to find etcd cert or key paths") 328 | return fmt.Errorf("Failed to find etcd cert or key paths") 329 | } 330 | 331 | s3Backup := c.Bool("s3-backup") 332 | bc := &backupConfig{ 333 | Backup: s3Backup, 334 | Endpoint: c.String("s3-endpoint"), 335 | AccessKey: c.String("s3-accessKey"), 336 | SecretKey: c.String("s3-secretKey"), 337 | BucketName: c.String("s3-bucketName"), 338 | Region: c.String("s3-region"), 339 | EndpointCA: c.String("s3-endpoint-ca"), 340 | Folder: c.String("s3-folder"), 341 | } 342 | 343 | if c.Bool("once") { 344 | backupName := c.String("name") 345 | 346 | log.WithFields(log.Fields{ 347 | "name": backupName, 348 | }).Info("Initializing Onetime Backup") 349 | 350 | compressedFilePath, err := CreateBackup(backupName, etcdCACert, etcdCert, etcdKey, etcdEndpoints, backupRetries) 351 | if err != nil { 352 | return err 353 | } 354 | if bc.Backup { 355 | err = CreateS3Backup(backupName, compressedFilePath, bc) 356 | if err != nil { 357 | return err 358 | } 359 | } 360 | prefix := getNamePrefix(backupName) 361 | // we only clean named backups if we have a retention period and a cluster name prefix 362 | if retentionPeriod != 0 && len(prefix) != 0 { 363 | if err := DeleteNamedBackups(retentionPeriod, prefix); err != nil { 364 | return err 365 | } 366 | } 367 | return nil 368 | } 369 | log.WithFields(log.Fields{ 370 | "creation": creationPeriod, 371 | "retention": retentionPeriod, 372 | }).Info("Initializing Rolling Backups") 373 | 374 | backupTicker := time.NewTicker(creationPeriod) 375 | for { 376 | select { 377 | case backupTime := <-backupTicker.C: 378 | backupName := fmt.Sprintf("%s_etcd", backupTime.Format(time.RFC3339)) 379 | err := retrieveAndWriteStatefile(backupName) 380 | if err != nil { 381 | // An error on statefile retrieval is not a reason to bail out 382 | // Having a snapshot without a statefile is more valuable than not having a snapshot at all 383 | log.WithFields(log.Fields{ 384 | "name": backupName, 385 | "error": err, 386 | }).Warn("Error while trying to retrieve cluster state from cluster") 387 | } 388 | compressedFilePath, err := CreateBackup(backupName, etcdCACert, etcdCert, etcdKey, etcdEndpoints, backupRetries) 389 | if err != nil { 390 | continue 391 | } 392 | DeleteBackups(backupTime, retentionPeriod) 393 | if !bc.Backup { 394 | continue 395 | } 396 | err = CreateS3Backup(backupName, compressedFilePath, bc) 397 | if err != nil { 398 | continue 399 | } 400 | DeleteS3Backups(backupTime, retentionPeriod, bc) 401 | } 402 | } 403 | } 404 | 405 | func minioClientFromConfig(bc *backupConfig) (*minio.Client, error) { 406 | client, err := setS3Service(bc, true) 407 | if err != nil { 408 | log.WithFields(log.Fields{ 409 | "s3-endpoint": bc.Endpoint, 410 | "s3-bucketName": bc.BucketName, 411 | "s3-accessKey": bc.AccessKey, 412 | "s3-region": bc.Region, 413 | "s3-endpoint-ca": bc.EndpointCA, 414 | "s3-folder": bc.Folder, 415 | }).Errorf("failed to set s3 server: %s", err) 416 | return nil, fmt.Errorf("failed to set s3 server: %+v", err) 417 | } 418 | return client, nil 419 | } 420 | 421 | func CreateBackup(backupName, etcdCACert, etcdCert, etcdKey, endpoints string, backupRetries uint) (compressedFilePath string, err error) { 422 | backupFile := fmt.Sprintf("%s/%s", backupBaseDir, backupName) 423 | stateFile := fmt.Sprintf("%s/%s.%s", k8sBaseDir, backupName, clusterStateExtension) 424 | var data []byte 425 | for retries := uint(0); retries <= backupRetries; retries++ { 426 | if retries > 0 { 427 | time.Sleep(failureInterval) 428 | } 429 | // check if the cluster is healthy 430 | cmd := exec.Command("etcdctl", 431 | fmt.Sprintf("--endpoints=%s", endpoints), 432 | "--cacert="+etcdCACert, 433 | "--cert="+etcdCert, 434 | "--key="+etcdKey, 435 | "endpoint", "health") 436 | data, err = cmd.CombinedOutput() 437 | 438 | if strings.Contains(string(data), "unhealthy") { 439 | log.WithFields(log.Fields{ 440 | "error": err, 441 | "data": string(data), 442 | }).Warn("Checking member health failed from etcd member") 443 | err = fmt.Errorf("%s: %v", err, string(data)) 444 | continue 445 | } 446 | 447 | cmd = exec.Command("etcdctl", 448 | fmt.Sprintf("--endpoints=%s", endpoints), 449 | "--cacert="+etcdCACert, 450 | "--cert="+etcdCert, 451 | "--key="+etcdKey, 452 | "snapshot", "save", backupFile) 453 | 454 | startTime := time.Now() 455 | data, err = cmd.CombinedOutput() 456 | endTime := time.Now() 457 | 458 | if err != nil { 459 | log.WithFields(log.Fields{ 460 | "attempt": retries + 1, 461 | "error": err, 462 | "data": string(data), 463 | }).Warn("Backup failed") 464 | err = fmt.Errorf("%s: %v", err, string(data)) 465 | continue 466 | } 467 | // Determine how many files need to be in the compressed file 468 | // 1. the compressed file 469 | toCompressFiles := []string{backupFile} 470 | // 2. the state file if present 471 | if _, err = os.Stat(stateFile); err == nil { 472 | toCompressFiles = append(toCompressFiles, stateFile) 473 | } 474 | // Create compressed file 475 | compressedFilePath, err = compressFiles(backupFile, toCompressFiles) 476 | if err != nil { 477 | log.WithFields(log.Fields{ 478 | "attempt": retries + 1, 479 | "error": err, 480 | "data": string(data), 481 | }).Warn("Compressing backup failed") 482 | continue 483 | } 484 | // Remove the original file after successfully compressing it 485 | err = os.Remove(backupFile) 486 | if err != nil { 487 | log.WithFields(log.Fields{ 488 | "attempt": retries + 1, 489 | "error": err, 490 | "data": string(data), 491 | }).Warn("Removing uncompressed snapshot file failed") 492 | continue 493 | 494 | } 495 | // Remove the state file after successfully compressing it 496 | if _, err = os.Stat(stateFile); err == nil { 497 | err = os.Remove(stateFile) 498 | if err != nil { 499 | log.WithFields(log.Fields{ 500 | "attempt": retries + 1, 501 | "error": err, 502 | "data": string(data), 503 | }).Warn("Removing statefile failed") 504 | } 505 | } 506 | 507 | log.WithFields(log.Fields{ 508 | "name": backupName, 509 | "runtime": endTime.Sub(startTime), 510 | }).Info("Created local backup") 511 | 512 | if err = os.Chmod(compressedFilePath, 0600); err != nil { 513 | log.WithFields(log.Fields{ 514 | "attempt": retries + 1, 515 | "error": err, 516 | "data": string(data), 517 | }).Warn("changing permission of the compressed snapshot failed") 518 | continue 519 | } 520 | break 521 | } 522 | return 523 | } 524 | 525 | func CreateS3Backup(backupName, compressedFilePath string, bc *backupConfig) error { 526 | // If the minio client doesn't work now, it won't after retrying 527 | client, err := minioClientFromConfig(bc) 528 | if err != nil { 529 | return err 530 | } 531 | compressedFile := filepath.Base(compressedFilePath) 532 | // If folder is specified, prefix the file with the folder 533 | if len(bc.Folder) != 0 { 534 | compressedFile = fmt.Sprintf("%s/%s", bc.Folder, compressedFile) 535 | } 536 | // check if it exists already in the bucket, and if versioning is disabled on the bucket. If an error is detected, 537 | // assume we aren't privy to that information and do multiple uploads anyway. 538 | info, _ := client.StatObject(context.TODO(), bc.BucketName, compressedFile, minio.StatObjectOptions{}) 539 | if info.Size != 0 { 540 | versioning, _ := client.GetBucketVersioning(context.TODO(), bc.BucketName) 541 | if !versioning.Enabled() { 542 | log.WithFields(log.Fields{ 543 | "name": backupName, 544 | }).Info("Skipping upload to s3 because snapshot already exists and versioning is not enabled for the bucket") 545 | return nil 546 | } 547 | } 548 | 549 | err = uploadBackupFile(client, bc.BucketName, compressedFile, compressedFilePath, s3Retries) 550 | if err != nil { 551 | return err 552 | } 553 | return nil 554 | } 555 | 556 | func DeleteBackups(backupTime time.Time, retentionPeriod time.Duration) { 557 | files, err := os.ReadDir(backupBaseDir) 558 | if err != nil { 559 | log.WithFields(log.Fields{ 560 | "dir": backupBaseDir, 561 | "error": err, 562 | }).Warn("Can't read backup directory") 563 | } 564 | 565 | cutoffTime := backupTime.Add(retentionPeriod * -1) 566 | 567 | for _, file := range files { 568 | if file.IsDir() { 569 | log.WithFields(log.Fields{ 570 | "name": file.Name(), 571 | }).Warn("Ignored directory, expecting file") 572 | continue 573 | } 574 | 575 | backupTime, err2 := time.Parse(time.RFC3339, strings.Split(file.Name(), "_")[0]) 576 | if err2 != nil { 577 | log.WithFields(log.Fields{ 578 | "name": file.Name(), 579 | "error": err2, 580 | }).Warn("Couldn't parse backup") 581 | 582 | } else if backupTime.Before(cutoffTime) { 583 | _ = deleteBackup(file.Name()) 584 | } 585 | } 586 | } 587 | 588 | func deleteBackup(fileName string) error { 589 | toDelete := fmt.Sprintf("%s/%s", backupBaseDir, path.Base(fileName)) 590 | 591 | cmd := exec.Command("rm", "-f", toDelete) 592 | 593 | startTime := time.Now() 594 | err2 := cmd.Run() 595 | endTime := time.Now() 596 | 597 | if err2 != nil { 598 | log.WithFields(log.Fields{ 599 | "name": fileName, 600 | "error": err2, 601 | }).Warn("Delete local backup failed") 602 | return err2 603 | } 604 | log.WithFields(log.Fields{ 605 | "name": fileName, 606 | "runtime": endTime.Sub(startTime), 607 | }).Info("Deleted local backup") 608 | return nil 609 | } 610 | 611 | func DeleteS3Backups(backupTime time.Time, retentionPeriod time.Duration, bc *backupConfig) { 612 | log.WithFields(log.Fields{ 613 | "retention": retentionPeriod, 614 | }).Info("Invoking delete s3 backup files") 615 | var backupDeleteList []string 616 | client, err := minioClientFromConfig(bc) 617 | if err != nil { 618 | // An error on setting minio client is not a reason to bail out 619 | // Having a snapshot without an upload to s3 is more valuable than not having a snapshot at all 620 | log.WithFields(log.Fields{ 621 | "error": err, 622 | }).Warn("Error while trying to configure minio client") 623 | return 624 | } 625 | 626 | cutoffTime := backupTime.Add(retentionPeriod * -1) 627 | 628 | isRecursive := false 629 | prefix := "" 630 | if len(bc.Folder) != 0 { 631 | prefix = bc.Folder 632 | // Recurse will show us the files in the folder 633 | isRecursive = true 634 | } 635 | objectCh := client.ListObjects(context.TODO(), bc.BucketName, minio.ListObjectsOptions{ 636 | Prefix: prefix, 637 | Recursive: isRecursive, 638 | }) 639 | re := regexp.MustCompile(fmt.Sprintf(".+_etcd(|.%s)$", compressedExtension)) 640 | for object := range objectCh { 641 | if object.Err != nil { 642 | log.Error("error to fetch s3 file:", object.Err) 643 | return 644 | } 645 | // only parse backup file names that matches *_etcd format 646 | if re.MatchString(object.Key) { 647 | filename := object.Key 648 | 649 | if len(bc.Folder) != 0 { 650 | // example object.Key with folder: folder/timestamp_etcd.zip 651 | // folder and separator needs to be stripped so time can be parsed below 652 | log.Debugf("Stripping [%s] from [%s]", fmt.Sprintf("%s/", prefix), filename) 653 | filename = strings.TrimPrefix(filename, fmt.Sprintf("%s/", prefix)) 654 | } 655 | log.Debugf("object.Key: [%s], filename: [%s]", object.Key, filename) 656 | 657 | backupTime, err := time.Parse(time.RFC3339, strings.Split(filename, "_")[0]) 658 | if err != nil { 659 | log.WithFields(log.Fields{ 660 | "name": filename, 661 | "objectKey": object.Key, 662 | "error": err, 663 | }).Warn("Couldn't parse s3 backup") 664 | 665 | } else if backupTime.Before(cutoffTime) { 666 | // We use object.Key here as we need the full path when a folder is used 667 | log.Debugf("Adding [%s] to files to delete, backupTime: [%q], cutoffTime: [%q]", object.Key, backupTime, cutoffTime) 668 | backupDeleteList = append(backupDeleteList, object.Key) 669 | } 670 | } 671 | } 672 | log.Debugf("Found %d files to delete", len(backupDeleteList)) 673 | 674 | for i := range backupDeleteList { 675 | log.Infof("Start to delete s3 backup file [%s]", backupDeleteList[i]) 676 | err := client.RemoveObject(context.TODO(), bc.BucketName, backupDeleteList[i], minio.RemoveObjectOptions{}) 677 | if err != nil { 678 | log.Errorf("Error detected during deletion: %v", err) 679 | } else { 680 | log.Infof("Success delete s3 backup file [%s]", backupDeleteList[i]) 681 | } 682 | } 683 | } 684 | 685 | func DeleteBackupAction(c *cli.Context) error { 686 | name := c.String("name") 687 | if name == "" { 688 | return fmt.Errorf("snapshot name is required") 689 | } 690 | compressedPath := fmt.Sprintf("/backup/%s.%s", name, compressedExtension) 691 | uncompressedPath := fmt.Sprintf("/backup/%s", name) 692 | 693 | // Since we have to support compressed and uncompressed versions of snapshots. 694 | // We can't remove the uncompressed snapshot during cleanup unless we are 695 | // sure the compressed is there, hence the complex check. 696 | if c.Bool("cleanup") { 697 | if _, err := os.Stat(compressedPath); err == nil { 698 | // for cleanup, we only want to delete the uncompressed snapshot. 699 | // we don't need to go to s3 700 | return deleteBackup(uncompressedPath) 701 | } 702 | } else { 703 | for _, p := range []string{compressedPath, uncompressedPath} { 704 | if err := deleteBackup(p); err != nil { 705 | return err 706 | } 707 | } 708 | } 709 | 710 | if !c.Bool("s3-backup") { 711 | return nil 712 | } 713 | 714 | bc := &backupConfig{ 715 | Endpoint: c.String("s3-endpoint"), 716 | AccessKey: c.String("s3-accessKey"), 717 | SecretKey: c.String("s3-secretKey"), 718 | BucketName: c.String("s3-bucketName"), 719 | Region: c.String("s3-region"), 720 | EndpointCA: c.String("s3-endpoint-ca"), 721 | Folder: c.String("s3-folder"), 722 | } 723 | client, err := setS3Service(bc, true) 724 | if err != nil { 725 | log.WithFields(log.Fields{ 726 | "s3-endpoint": bc.Endpoint, 727 | "s3-bucketName": bc.BucketName, 728 | "s3-accessKey": bc.AccessKey, 729 | "s3-region": bc.Region, 730 | "s3-endpoint-ca": bc.EndpointCA, 731 | "s3-folder": bc.Folder, 732 | }).Errorf("failed to set s3 server: %s", err) 733 | return fmt.Errorf("failed to set s3 server: %+v", err) 734 | } 735 | folder := c.String("s3-folder") 736 | if len(folder) != 0 { 737 | name = fmt.Sprintf("%s/%s", folder, name) 738 | } 739 | 740 | doneCh := make(chan struct{}) 741 | defer close(doneCh) 742 | // list objects with prefix=name, this will include uncompressed and compressed backup objects 743 | objectCh := client.ListObjects(context.TODO(), bc.BucketName, minio.ListObjectsOptions{ 744 | Prefix: name, 745 | Recursive: false, 746 | }) 747 | var removed []string 748 | for object := range objectCh { 749 | if object.Err != nil { 750 | log.Errorf("failed to list objects in backup buckets [%s]: %v", bc.BucketName, object.Err) 751 | return object.Err 752 | } 753 | log.Infof("deleting object with key: %s that matches prefix: %s", object.Key, name) 754 | err = client.RemoveObject(context.TODO(), bc.BucketName, object.Key, minio.RemoveObjectOptions{}) 755 | if err != nil { 756 | return err 757 | } 758 | removed = append(removed, object.Key) 759 | } 760 | 761 | log.Infof("removed backups: %s from object store", strings.Join(removed, ", ")) 762 | 763 | return nil 764 | } 765 | 766 | func setS3Service(bc *backupConfig, useSSL bool) (*minio.Client, error) { 767 | // Initialize minio client object. 768 | log.WithFields(log.Fields{ 769 | "s3-endpoint": bc.Endpoint, 770 | "s3-bucketName": bc.BucketName, 771 | "s3-accessKey": bc.AccessKey, 772 | "s3-region": bc.Region, 773 | "s3-endpoint-ca": bc.EndpointCA, 774 | "s3-folder": bc.Folder, 775 | }).Info("invoking set s3 service client") 776 | 777 | var err error 778 | var client = &minio.Client{} 779 | var cred *credentials.Credentials 780 | var tr = http.DefaultTransport 781 | if bc.EndpointCA != "" { 782 | tr, err = setTransportCA(tr, bc.EndpointCA) 783 | if err != nil { 784 | return nil, err 785 | } 786 | } 787 | bucketLookup := getBucketLookupType(bc.Endpoint) 788 | for retries := 0; retries <= defaultS3Retries; retries++ { 789 | // if the s3 access key and secret is not set use iam role 790 | if len(bc.AccessKey) == 0 && len(bc.SecretKey) == 0 { 791 | log.Info("invoking set s3 service client use IAM role") 792 | cred = credentials.NewIAM("") 793 | if bc.Endpoint == "" { 794 | bc.Endpoint = s3Endpoint 795 | } 796 | } else { 797 | // Base64 decoding S3 accessKey and secretKey before create static credentials 798 | // To be backward compatible, just updating base64 encoded values 799 | accessKey := bc.AccessKey 800 | secretKey := bc.SecretKey 801 | if len(accessKey) > 0 { 802 | v, err := base64.StdEncoding.DecodeString(accessKey) 803 | if err == nil { 804 | accessKey = string(v) 805 | } 806 | } 807 | if len(secretKey) > 0 { 808 | v, err := base64.StdEncoding.DecodeString(secretKey) 809 | if err == nil { 810 | secretKey = string(v) 811 | } 812 | } 813 | cred = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureDefault) 814 | } 815 | client, err = minio.New(bc.Endpoint, &minio.Options{ 816 | Creds: cred, 817 | Secure: useSSL, 818 | Region: bc.Region, 819 | BucketLookup: bucketLookup, 820 | Transport: tr, 821 | }) 822 | if err != nil { 823 | log.Infof("failed to init s3 client server: %v, retried %d times", err, retries) 824 | if retries >= defaultS3Retries { 825 | return nil, fmt.Errorf("failed to set s3 server: %v", err) 826 | } 827 | continue 828 | } 829 | 830 | break 831 | } 832 | 833 | found, err := client.BucketExists(context.TODO(), bc.BucketName) 834 | if err != nil { 835 | return nil, fmt.Errorf("failed to check s3 bucket:%s, err:%v", bc.BucketName, err) 836 | } 837 | if !found { 838 | return nil, fmt.Errorf("bucket %s is not found", bc.BucketName) 839 | } 840 | return client, nil 841 | } 842 | 843 | func getBucketLookupType(endpoint string) minio.BucketLookupType { 844 | if endpoint == "" { 845 | return minio.BucketLookupAuto 846 | } 847 | if strings.Contains(endpoint, "aliyun") { 848 | return minio.BucketLookupDNS 849 | } 850 | return minio.BucketLookupAuto 851 | } 852 | 853 | func uploadBackupFile(svc *minio.Client, bucketName, fileName, filePath string, s3Retries uint) error { 854 | var info minio.UploadInfo 855 | var err error 856 | // Upload the zip file with FPutObject 857 | log.Infof("invoking uploading backup file [%s] to s3", fileName) 858 | for i := uint(0); i <= s3Retries; i++ { 859 | info, err = svc.FPutObject(context.TODO(), bucketName, fileName, filePath, minio.PutObjectOptions{ContentType: contentType}) 860 | if err == nil { 861 | log.Infof("Successfully uploaded [%s] of size [%d]", fileName, info.Size) 862 | return nil 863 | } 864 | log.Infof("failed to upload etcd snapshot file: %v, retried %d times", err, i) 865 | } 866 | return fmt.Errorf("failed to upload etcd snapshot file: %v", err) 867 | } 868 | 869 | func DownloadBackupAction(c *cli.Context) error { 870 | log.Info("Initializing Download Backups") 871 | SetLoggingLevel(c.Bool("debug")) 872 | if c.Bool("s3-backup") { 873 | return DownloadS3Backup(c) 874 | } 875 | return DownloadLocalBackup(c) 876 | } 877 | 878 | func ExtractStateFileAction(c *cli.Context) error { 879 | SetLoggingLevel(c.Bool("debug")) 880 | name := path.Base(c.String("name")) 881 | log.Infof("Trying to get statefile from backup [%s]", name) 882 | if c.Bool("s3-backup") { 883 | err := DownloadS3Backup(c) 884 | if err != nil { 885 | return err 886 | } 887 | } 888 | // Destination filename for statefile 889 | stateFilePath := fmt.Sprintf("%s/%s.%s", k8sBaseDir, name, clusterStateExtension) 890 | // Location of the compressed snapshot file 891 | compressedFilePath := fmt.Sprintf("/backup/%s.%s", name, compressedExtension) 892 | // Check if compressed snapshot file exists 893 | if _, err := os.Stat(compressedFilePath); err != nil { 894 | return err 895 | } 896 | // Extract statefile content in archive 897 | err := decompressFile(compressedFilePath, stateFilePath, tmpStateFilePath) 898 | if err != nil { 899 | return fmt.Errorf("Unable to extract file [%s] from file [%s] to destination [%s]: %v", stateFilePath, compressedFilePath, tmpStateFilePath, err) 900 | } 901 | log.Infof("Successfully extracted file [%s] from file [%s] to destination [%s]", stateFilePath, compressedFilePath, tmpStateFilePath) 902 | 903 | return nil 904 | } 905 | 906 | func DownloadS3Backup(c *cli.Context) error { 907 | bc := &backupConfig{ 908 | Endpoint: c.String("s3-endpoint"), 909 | AccessKey: c.String("s3-accessKey"), 910 | SecretKey: c.String("s3-secretKey"), 911 | BucketName: c.String("s3-bucketName"), 912 | Region: c.String("s3-region"), 913 | EndpointCA: c.String("s3-endpoint-ca"), 914 | Folder: c.String("s3-folder"), 915 | } 916 | client, err := setS3Service(bc, true) 917 | if err != nil { 918 | log.WithFields(log.Fields{ 919 | "s3-endpoint": bc.Endpoint, 920 | "s3-bucketName": bc.BucketName, 921 | "s3-accessKey": bc.AccessKey, 922 | "s3-region": bc.Region, 923 | "s3-endpoint-ca": bc.EndpointCA, 924 | "s3-folder": bc.Folder, 925 | }).Errorf("failed to set s3 server: %s", err) 926 | return fmt.Errorf("failed to set s3 server: %+v", err) 927 | } 928 | 929 | prefix := c.String("name") 930 | if len(prefix) == 0 { 931 | return fmt.Errorf("empty backup name") 932 | } 933 | folder := c.String("s3-folder") 934 | if len(folder) != 0 { 935 | prefix = fmt.Sprintf("%s/%s", folder, prefix) 936 | } 937 | // we need download with prefix because we don't know if the file is ziped or not 938 | filename, err := downloadFromS3WithPrefix(client, prefix, bc.BucketName) 939 | if err != nil { 940 | return err 941 | } 942 | if isCompressed(filename) { 943 | log.Infof("Decompressing etcd snapshot file [%s]", filename) 944 | compressedFilePath := fmt.Sprintf("%s/%s", backupBaseDir, filename) 945 | fileLocation := fmt.Sprintf("%s/%s", backupBaseDir, decompressedName(filename)) 946 | err := decompressFile(compressedFilePath, fileLocation, fileLocation) 947 | if err != nil { 948 | return fmt.Errorf("Unable to decompress [%s] to [%s]: %v", compressedFilePath, fileLocation, err) 949 | } 950 | 951 | log.Infof("Decompressed [%s] to [%s]", compressedFilePath, fileLocation) 952 | } 953 | return nil 954 | } 955 | 956 | func DownloadLocalBackup(c *cli.Context) error { 957 | snapshot := path.Base(c.String("name")) 958 | endpoint := c.String("local-endpoint") 959 | if snapshot == "." || snapshot == "/" { 960 | return fmt.Errorf("snapshot name is required") 961 | } 962 | if len(endpoint) == 0 { 963 | return fmt.Errorf("local-endpoint is required") 964 | } 965 | certs, err := getCertsFromCli(c) 966 | if err != nil { 967 | return err 968 | } 969 | tlsConfig, err := setupTLSConfig(certs, false) 970 | if err != nil { 971 | return err 972 | } 973 | client := http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig}} 974 | snapshotURL := fmt.Sprintf("https://%s:%s/%s", endpoint, serverPort, snapshot) 975 | log.Infof("Invoking downloading backup files: %s", snapshot) 976 | log.Infof("Trying to download backup file from: %s", snapshotURL) 977 | resp, err := client.Get(snapshotURL) 978 | if err != nil { 979 | return err 980 | } 981 | if resp.StatusCode != http.StatusOK { 982 | log.Errorf("backup download failed: %v", resp.Body) 983 | return fmt.Errorf("backup download failed: %v", resp.Body) 984 | } 985 | defer resp.Body.Close() 986 | 987 | snapshotFileLocation := fmt.Sprintf("%s/%s", backupBaseDir, snapshot) 988 | snapshotFile, err := os.Create(snapshotFileLocation) 989 | if err != nil { 990 | return err 991 | } 992 | defer snapshotFile.Close() 993 | 994 | if _, err := io.Copy(snapshotFile, resp.Body); err != nil { 995 | return err 996 | } 997 | 998 | if err := os.Chmod(snapshotFileLocation, 0600); err != nil { 999 | log.WithFields(log.Fields{ 1000 | "error": err, 1001 | }).Warn("changing permission of the locally downloaded snapshot failed") 1002 | } 1003 | 1004 | log.Infof("Successfully download %s from %s ", snapshot, endpoint) 1005 | return nil 1006 | } 1007 | 1008 | func DeleteNamedBackups(retentionPeriod time.Duration, prefix string) error { 1009 | files, err := os.ReadDir(backupBaseDir) 1010 | if err != nil { 1011 | log.WithFields(log.Fields{ 1012 | "dir": backupBaseDir, 1013 | "error": err, 1014 | }).Warn("Can't read backup directory") 1015 | return err 1016 | } 1017 | cutoffTime := time.Now().Add(retentionPeriod * -1) 1018 | for _, file := range files { 1019 | fi, err := file.Info() 1020 | if err != nil { 1021 | return fmt.Errorf("failed to get file info: %w", err) 1022 | } 1023 | if strings.HasPrefix(file.Name(), prefix) && fi.ModTime().Before(cutoffTime) && IsRecurringSnapshot(file.Name()) { 1024 | if err = deleteBackup(file.Name()); err != nil { 1025 | return err 1026 | } 1027 | } 1028 | } 1029 | return nil 1030 | } 1031 | 1032 | func getNamePrefix(name string) string { 1033 | re := regexp.MustCompile("^c-[a-z0-9].*?-") 1034 | m := re.FindStringSubmatch(name) 1035 | if len(m) == 0 { 1036 | return "" 1037 | } 1038 | return m[0] 1039 | } 1040 | 1041 | func ServeBackupAction(c *cli.Context) error { 1042 | snapshot := path.Base(c.String("name")) 1043 | 1044 | if snapshot == "." || snapshot == "/" { 1045 | return fmt.Errorf("snapshot name is required") 1046 | } 1047 | // Check if snapshot is compressed 1048 | compressedFilePath := fmt.Sprintf("%s/%s.%s", backupBaseDir, snapshot, compressedExtension) 1049 | fileLocation := fmt.Sprintf("%s/%s", backupBaseDir, snapshot) 1050 | if _, err := os.Stat(compressedFilePath); err == nil { 1051 | err := decompressFile(compressedFilePath, fileLocation, fileLocation) 1052 | if err != nil { 1053 | return err 1054 | } 1055 | log.Infof("Extracted from %s", compressedFilePath) 1056 | } 1057 | 1058 | if _, err := os.Stat(fmt.Sprintf("%s/%s", backupBaseDir, snapshot)); err != nil { 1059 | return err 1060 | } 1061 | certs, err := getCertsFromCli(c) 1062 | if err != nil { 1063 | return err 1064 | } 1065 | tlsConfig, err := setupTLSConfig(certs, true) 1066 | if err != nil { 1067 | return err 1068 | } 1069 | httpServer := &http.Server{ 1070 | Addr: fmt.Sprintf("0.0.0.0:%s", serverPort), 1071 | TLSConfig: tlsConfig, 1072 | } 1073 | 1074 | http.HandleFunc(fmt.Sprintf("/%s", snapshot), func(response http.ResponseWriter, request *http.Request) { 1075 | http.ServeFile(response, request, fmt.Sprintf("%s/%s", backupBaseDir, snapshot)) 1076 | }) 1077 | return httpServer.ListenAndServeTLS(certs["cert"], certs["key"]) 1078 | } 1079 | 1080 | func getCertsFromCli(c *cli.Context) (map[string]string, error) { 1081 | caCert := c.String("cacert") 1082 | cert := c.String("cert") 1083 | key := c.String("key") 1084 | if len(cert) == 0 || len(caCert) == 0 || len(key) == 0 { 1085 | return nil, fmt.Errorf("cacert, cert and key are required") 1086 | } 1087 | 1088 | return map[string]string{"cacert": caCert, "cert": cert, "key": key}, nil 1089 | } 1090 | 1091 | func setupTLSConfig(certs map[string]string, isServer bool) (*tls.Config, error) { 1092 | caCertPem, err := os.ReadFile(certs["cacert"]) 1093 | if err != nil { 1094 | return nil, err 1095 | } 1096 | tlsConfig := &tls.Config{} 1097 | certPool := x509.NewCertPool() 1098 | certPool.AppendCertsFromPEM(caCertPem) 1099 | if isServer { 1100 | tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert 1101 | tlsConfig.ClientCAs = certPool 1102 | tlsConfig.MinVersion = tls.VersionTLS12 1103 | } else { // client config 1104 | x509Pair, err := tls.LoadX509KeyPair(certs["cert"], certs["key"]) 1105 | if err != nil { 1106 | return nil, err 1107 | } 1108 | tlsConfig.Certificates = []tls.Certificate{x509Pair} 1109 | tlsConfig.RootCAs = certPool 1110 | // This is to avoid IP SAN errors. 1111 | tlsConfig.InsecureSkipVerify = true 1112 | } 1113 | 1114 | tlsConfig.BuildNameToCertificate() 1115 | return tlsConfig, nil 1116 | } 1117 | 1118 | func IsRecurringSnapshot(name string) bool { 1119 | // name is fmt.Sprintf("%s-%s%s-", cluster.Name, typeFlag, providerFlag) 1120 | // typeFlag = "r": recurring 1121 | // typeFlag = "m": manual 1122 | // 1123 | // providerFlag = "l" local 1124 | // providerFlag = "s" s3 1125 | re := regexp.MustCompile("^c-[a-z0-9].*?-r.-") 1126 | return re.MatchString(name) 1127 | } 1128 | 1129 | func downloadFromS3WithPrefix(client *minio.Client, prefix, bucket string) (string, error) { 1130 | var filename string 1131 | 1132 | objectCh := client.ListObjects(context.TODO(), bucket, minio.ListObjectsOptions{ 1133 | Prefix: prefix, 1134 | Recursive: false, 1135 | }) 1136 | for object := range objectCh { 1137 | if object.Err != nil { 1138 | log.Errorf("failed to list objects in backup buckets [%s]: %v", bucket, object.Err) 1139 | return "", object.Err 1140 | } 1141 | decompressedFilename := decompressedName(object.Key) 1142 | log.Debugf("found key: [%s], decompressedFilename: [%s]", object.Key, decompressedFilename) 1143 | if prefix == decompressedFilename { 1144 | filename = object.Key 1145 | break 1146 | } 1147 | decodedDecompressedFilename, err := url.QueryUnescape(decompressedFilename) 1148 | if err != nil { 1149 | log.Errorf("Unable to decode filename [%s]: %v", decompressedFilename, err) 1150 | continue 1151 | } 1152 | if prefix == decodedDecompressedFilename { 1153 | decodedObjectKey, err := url.QueryUnescape(object.Key) 1154 | if err != nil { 1155 | log.Errorf("Unable to decode object.Key [%s]: %v", object.Key, err) 1156 | continue 1157 | } 1158 | filename = decodedObjectKey 1159 | break 1160 | } 1161 | } 1162 | if len(filename) == 0 { 1163 | return "", fmt.Errorf("failed to download s3 backup: no backups found") 1164 | } 1165 | // if folder is included, strip it so it doesn't end up in a folder on the host itself 1166 | targetFilename := path.Base(filename) 1167 | targetFileLocation := fmt.Sprintf("%s/%s", backupBaseDir, targetFilename) 1168 | var object *minio.Object 1169 | var err error 1170 | 1171 | for retries := 0; retries <= defaultS3Retries; retries++ { 1172 | object, err = client.GetObject(context.TODO(), bucket, filename, minio.GetObjectOptions{}) 1173 | if err != nil { 1174 | log.Infof("Failed to download etcd snapshot file [%s]: %v, retried %d times", filename, err, retries) 1175 | if retries >= defaultS3Retries { 1176 | return "", fmt.Errorf("Unable to download backup file for [%s]: %v", filename, err) 1177 | } 1178 | } 1179 | log.Infof("Successfully downloaded [%s]", filename) 1180 | } 1181 | 1182 | localFile, err := os.Create(targetFileLocation) 1183 | if err != nil { 1184 | return "", fmt.Errorf("Failed to create local file [%s]: %v", targetFileLocation, err) 1185 | } 1186 | defer localFile.Close() 1187 | 1188 | if _, err = io.Copy(localFile, object); err != nil { 1189 | return "", fmt.Errorf("Failed to copy retrieved object to local file [%s]: %v", targetFileLocation, err) 1190 | } 1191 | if err := os.Chmod(targetFileLocation, 0600); err != nil { 1192 | return "", fmt.Errorf("changing permission of the locally downloaded snapshot failed") 1193 | } 1194 | 1195 | return targetFilename, nil 1196 | } 1197 | 1198 | func compressFiles(destinationFile string, fileNames []string) (string, error) { 1199 | // Create destination file 1200 | compressedFile := fmt.Sprintf("%s.%s", destinationFile, compressedExtension) 1201 | zipFile, err := os.Create(compressedFile) 1202 | if err != nil { 1203 | return "", err 1204 | } 1205 | defer zipFile.Close() 1206 | 1207 | zipWriter := zip.NewWriter(zipFile) 1208 | defer zipWriter.Close() 1209 | 1210 | for _, file := range fileNames { 1211 | if err = AddFileToZip(zipWriter, file); err != nil { 1212 | return "", err 1213 | } 1214 | } 1215 | return compressedFile, nil 1216 | } 1217 | 1218 | // decompressFile: Thanks to https://golangcode.com/unzip-files-in-go/ 1219 | func decompressFile(src string, filePath string, dest string) error { 1220 | 1221 | var fileFound bool 1222 | 1223 | r, err := zip.OpenReader(src) 1224 | if err != nil { 1225 | return err 1226 | } 1227 | defer r.Close() 1228 | 1229 | for _, f := range r.File { 1230 | if f.Name == filePath { 1231 | outFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) 1232 | if err != nil { 1233 | return err 1234 | } 1235 | 1236 | rc, err := f.Open() 1237 | if err != nil { 1238 | return err 1239 | } 1240 | 1241 | _, err = io.Copy(outFile, rc) 1242 | 1243 | // Close the file without defer to close before next iteration of loop 1244 | outFile.Close() 1245 | rc.Close() 1246 | 1247 | if err != nil { 1248 | return err 1249 | } 1250 | fileFound = true 1251 | break 1252 | } 1253 | } 1254 | 1255 | if fileFound { 1256 | if err := os.Chmod(dest, 0600); err != nil { 1257 | log.WithFields(log.Fields{ 1258 | "error": err, 1259 | }).Warn("changing permission of the decompressed snapshot failed") 1260 | } 1261 | return nil 1262 | } 1263 | return fmt.Errorf("File [%s] not found in file [%s]", filePath, src) 1264 | } 1265 | 1266 | func readS3EndpointCA(endpointCA string) ([]byte, error) { 1267 | // I expect the CA to be passed as base64 string OR a file system path. 1268 | // I do this to be able to pass it through rke/rancher api without writing it 1269 | // to the backup container filesystem. 1270 | ca, err := base64.StdEncoding.DecodeString(endpointCA) 1271 | if err == nil { 1272 | log.Debug("reading s3-endpoint-ca as a base64 string") 1273 | } else { 1274 | ca, err = os.ReadFile(endpointCA) 1275 | log.Debugf("reading s3-endpoint-ca from [%v]", endpointCA) 1276 | } 1277 | return ca, err 1278 | } 1279 | 1280 | func isValidCertificate(c []byte) bool { 1281 | p, _ := pem.Decode(c) 1282 | if p == nil { 1283 | return false 1284 | } 1285 | _, err := x509.ParseCertificates(p.Bytes) 1286 | return err == nil 1287 | } 1288 | 1289 | func setTransportCA(tr http.RoundTripper, endpointCA string) (http.RoundTripper, error) { 1290 | ca, err := readS3EndpointCA(endpointCA) 1291 | if err != nil { 1292 | return tr, err 1293 | } 1294 | if !isValidCertificate(ca) { 1295 | return tr, fmt.Errorf("s3-endpoint-ca is not a valid x509 certificate") 1296 | } 1297 | certPool := x509.NewCertPool() 1298 | certPool.AppendCertsFromPEM(ca) 1299 | 1300 | tr.(*http.Transport).TLSClientConfig = &tls.Config{ 1301 | RootCAs: certPool, 1302 | } 1303 | 1304 | return tr, nil 1305 | } 1306 | 1307 | func isCompressed(filename string) bool { 1308 | return strings.HasSuffix(filename, fmt.Sprintf(".%s", compressedExtension)) 1309 | } 1310 | 1311 | func decompressedName(filename string) string { 1312 | return strings.TrimSuffix(filename, path.Ext(filename)) 1313 | } 1314 | 1315 | func AddFileToZip(zipWriter *zip.Writer, filename string) error { 1316 | fileToZip, err := os.Open(filename) 1317 | if err != nil { 1318 | return err 1319 | } 1320 | defer fileToZip.Close() 1321 | 1322 | // Get the file information 1323 | info, err := fileToZip.Stat() 1324 | if err != nil { 1325 | return err 1326 | } 1327 | 1328 | header, err := zip.FileInfoHeader(info) 1329 | if err != nil { 1330 | return err 1331 | } 1332 | 1333 | // Using FileInfoHeader() above only uses the basename of the file. If we want 1334 | // to preserve the folder structure we can overwrite this with the full path. 1335 | header.Name = filename 1336 | 1337 | // Change to deflate to gain better compression 1338 | // see http://golang.org/pkg/archive/zip/#pkg-constants 1339 | header.Method = zip.Deflate 1340 | header.Modified = time.Unix(0, 0) 1341 | 1342 | writer, err := zipWriter.CreateHeader(header) 1343 | if err != nil { 1344 | return err 1345 | } 1346 | _, err = io.Copy(writer, fileToZip) 1347 | return err 1348 | } 1349 | 1350 | func retrieveAndWriteStatefile(backupName string) error { 1351 | log.WithFields(log.Fields{ 1352 | "name": backupName, 1353 | }).Debug("retrieveAndWriteStatefile called") 1354 | 1355 | var out bytes.Buffer 1356 | var err error 1357 | for retries := 0; retries <= defaultBackupRetries; retries++ { 1358 | log.WithFields(log.Fields{ 1359 | "attempt": retries + 1, 1360 | "name": backupName, 1361 | }).Info("Trying to retrieve secret full-cluster-state using kubectl") 1362 | 1363 | if retries > 0 { 1364 | time.Sleep(failureInterval) 1365 | } 1366 | 1367 | // Try to retrieve cluster state to include in snapshot 1368 | cmd := exec.Command("/usr/local/bin/kubectl", "--request-timeout=30s", "--kubeconfig", "/etc/kubernetes/ssl/kubecfg-kube-node.yaml", "-n", "kube-system", "get", "secret", "full-cluster-state", "-o", "json") 1369 | var stderr bytes.Buffer 1370 | cmd.Stdout = &out 1371 | cmd.Stderr = &stderr 1372 | err = cmd.Run() 1373 | if err != nil { 1374 | log.WithFields(log.Fields{ 1375 | "attempt": retries + 1, 1376 | "name": backupName, 1377 | "err": fmt.Sprintf("%s: %s", err, stderr.String()), 1378 | }).Warn("Failed to retrieve secret full-cluster-state using kubectl") 1379 | if retries >= defaultBackupRetries { 1380 | return fmt.Errorf("Failed to retrieve secret full-cluster-state using kubectl: %v", fmt.Sprintf("%s: %s", err, stderr.String())) 1381 | } 1382 | continue 1383 | } 1384 | break 1385 | } 1386 | var m map[string]interface{} 1387 | err = json.Unmarshal(out.Bytes(), &m) 1388 | if err != nil { 1389 | return fmt.Errorf("Failed to unmarshal cluster state from secret full-cluster-state: %v", err) 1390 | } 1391 | 1392 | // Extract the data field from the secret 1393 | var jsondata map[string]interface{} 1394 | var encodedFullClusterState string 1395 | if _, ok := m["data"]; ok { 1396 | jsondata = m["data"].(map[string]interface{}) 1397 | } 1398 | if str, ok := jsondata["full-cluster-state"].(string); ok { 1399 | encodedFullClusterState = str 1400 | } 1401 | 1402 | // Decode the base64-encoded full-cluster-state 1403 | fullClusterState, err := base64.StdEncoding.DecodeString(encodedFullClusterState) 1404 | if err != nil { 1405 | return fmt.Errorf("Failed to decode base64 full-cluster-state: %v", err) 1406 | } 1407 | 1408 | var prettyFullClusterState bytes.Buffer 1409 | err = json.Indent(&prettyFullClusterState, fullClusterState, "", " ") 1410 | if err != nil { 1411 | return fmt.Errorf("Failed to indent JSON for state file: %v", err) 1412 | } 1413 | stateFilePath := fmt.Sprintf("/etc/kubernetes/%s.rkestate", backupName) 1414 | f, err := os.Create(stateFilePath) 1415 | if err != nil { 1416 | return fmt.Errorf("Failed to create state file [%s]: %v", stateFilePath, err) 1417 | } 1418 | defer f.Close() 1419 | _, err = f.Write(prettyFullClusterState.Bytes()) 1420 | if err != nil { 1421 | return fmt.Errorf("Failed to write state file [%s]: %v", stateFilePath, err) 1422 | } 1423 | log.WithFields(log.Fields{ 1424 | "filepath": stateFilePath, 1425 | }).Info("Successfully written state file content to file") 1426 | 1427 | return nil 1428 | } 1429 | --------------------------------------------------------------------------------