├── .dockerignore ├── .github └── workflows │ ├── artifacts.yaml │ ├── golangci-lint.yaml │ └── test-e2e.yaml ├── .gitignore ├── .golangci.yml ├── CODEOWNERS ├── Dockerfile ├── Dockerfile.e2e ├── LICENSE ├── Makefile ├── PROJECT ├── README.md ├── api └── v1alpha1 │ ├── constants.go │ ├── default.go │ ├── distributedrediscluster_types.go │ ├── groupversion_info.go │ ├── redisclusterbackup_types.go │ ├── redisclustercleanup_types.go │ └── zz_generated.deepcopy.go ├── cmd └── main.go ├── config ├── crd │ ├── bases │ │ ├── redis.kun_distributedredisclusters.yaml │ │ ├── redis.kun_redisclusterbackups.yaml │ │ └── redis.kun_redisclustercleanups.yaml │ ├── kustomization.yaml │ └── kustomizeconfig.yaml ├── default │ ├── kustomization.yaml │ ├── manager_metrics_patch.yaml │ └── metrics_service.yaml ├── manager │ ├── kustomization.yaml │ └── manager.yaml ├── manifests │ └── kustomization.yaml ├── network-policy │ ├── allow-metrics-traffic.yaml │ └── kustomization.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── rbac │ ├── distributedrediscluster_editor_role.yaml │ ├── distributedrediscluster_viewer_role.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── metrics_auth_role.yaml │ ├── metrics_auth_role_binding.yaml │ ├── metrics_reader_role.yaml │ ├── redisclusterbackup_editor_role.yaml │ ├── redisclusterbackup_viewer_role.yaml │ ├── redisclustercleanup_editor_role.yaml │ ├── redisclustercleanup_viewer_role.yaml │ ├── role.yaml │ ├── role_binding.yaml │ └── service_account.yaml ├── samples │ ├── example │ │ ├── backup-restore │ │ │ ├── redisclusterbackup_cr.yaml │ │ │ ├── redisclusterbackup_topvc.yaml │ │ │ ├── restore.yaml │ │ │ └── restore_frompvc.yaml │ │ ├── custom-config.yaml │ │ ├── custom-password.yaml │ │ ├── custom-resources.yaml │ │ ├── custom-service.yaml │ │ ├── persistent.yaml │ │ ├── prometheus-exporter.yaml │ │ ├── redis.kun_v1alpha1_distributedrediscluster_cr.yaml │ │ └── securitycontext.yaml │ ├── kustomization.yaml │ ├── redis_v1alpha1_distributedrediscluster.yaml │ ├── redis_v1alpha1_redisclusterbackup.yaml │ └── redis_v1alpha1_redisclustercleanup.yaml └── scorecard │ ├── bases │ └── config.yaml │ ├── kustomization.yaml │ └── patches │ ├── basic.config.yaml │ └── olm.config.yaml ├── go.mod ├── go.sum ├── hack └── boilerplate.go.txt ├── internal ├── clustering │ ├── migration.go │ ├── migration_test.go │ ├── placement.go │ ├── placement_v2.go │ ├── rebalance.go │ ├── rebalance_test.go │ └── roles.go ├── config │ └── redis.go ├── controller │ ├── distributedrediscluster │ │ ├── distributedrediscluster_controller.go │ │ ├── helper.go │ │ ├── status.go │ │ └── sync_handler.go │ ├── errors.go │ ├── redisclusterbackup │ │ ├── helper.go │ │ ├── redisclusterbackup_controller.go │ │ └── sync_handler.go │ └── redisclustercleanup │ │ ├── cleaner.go │ │ └── redisclustercleanup_controller.go ├── event │ └── event.go ├── exec │ └── exec.go ├── heal │ ├── clustersplit.go │ ├── clustersplit_test.go │ ├── failednodes.go │ ├── heal.go │ ├── terminatingpod.go │ └── untrustenodes.go ├── k8sutil │ ├── batchcronjob.go │ ├── batchjob.go │ ├── configmap.go │ ├── customresource.go │ ├── pod.go │ ├── poddisruptionbudget.go │ ├── pvc.go │ ├── service.go │ ├── statefulset.go │ └── util.go ├── manager │ ├── checker.go │ ├── ensurer.go │ ├── ensurer_test.go │ └── healer.go ├── osm │ ├── context │ │ └── lib.go │ ├── google │ │ ├── config.go │ │ ├── container.go │ │ ├── item.go │ │ └── location.go │ ├── osm.go │ └── rclone.go ├── redisutil │ ├── admin.go │ ├── client.go │ ├── cluster.go │ ├── clusterinfo.go │ ├── connections.go │ ├── errors.go │ ├── node.go │ ├── node_test.go │ ├── slot.go │ └── slot_test.go ├── resources │ ├── configmaps │ │ ├── configmap.go │ │ └── configmap_test.go │ ├── poddisruptionbudgets │ │ └── poddisruptionbudget.go │ ├── services │ │ └── service.go │ └── statefulsets │ │ ├── helper.go │ │ ├── statefulset.go │ │ └── statefulset_test.go └── utils │ ├── compare.go │ ├── labels.go │ ├── math.go │ ├── parse.go │ ├── parse_test.go │ ├── rename_cmd.go │ ├── scoped.go │ ├── string.go │ └── types.go ├── static └── redis-cluster.png └── test ├── e2e ├── README.md ├── drc_crud │ ├── drc │ │ ├── client.go │ │ ├── framework.go │ │ ├── goredis_util.go │ │ ├── operator_util.go │ │ └── rename.conf │ ├── e2e_suite_test.go │ ├── e2e_test.go │ └── job.yaml ├── drc_operator │ ├── e2e_suite_test.go │ └── e2e_test.go └── drcb │ ├── drcb_suite_test.go │ └── drcb_test.go └── utils └── utils.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file 2 | # Ignore build and test binaries. 3 | bin/ 4 | -------------------------------------------------------------------------------- /.github/workflows/artifacts.yaml: -------------------------------------------------------------------------------- 1 | name: Build and push docker image 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v[0-9]+.[0-9]+.[0-9]+" 7 | 8 | jobs: 9 | check-base-branch: 10 | runs-on: ubuntu-latest 11 | outputs: 12 | on_main: ${{ steps.contains_tag.outputs.retval }} 13 | steps: 14 | # refer to https://github.com/rickstaa/action-contains-tag/pull/18 for more info 15 | - name: Workaround regression action-contains-tag due to git update 16 | run: git config --global remote.origin.followRemoteHEAD never 17 | 18 | - name: Git checkout 19 | uses: actions/checkout@v4 20 | with: 21 | fetch-depth: 0 22 | 23 | - uses: rickstaa/action-contains-tag@v1 24 | id: contains_tag 25 | with: 26 | reference: "main" 27 | tag: "${{ github.ref }}" 28 | 29 | build-and-push: 30 | runs-on: ubuntu-latest 31 | needs: check-base-branch 32 | if: ${{ needs.check-base-branch.outputs.on_main == 'true' }} 33 | steps: 34 | - name: Git checkout 35 | uses: actions/checkout@v4 36 | 37 | - uses: TykTechnologies/actions/docker-login@main 38 | with: 39 | dockerhub_username: ${{ secrets.DOCKER_USERNAME }} 40 | dockerhub_token: ${{ secrets.DOCKER_PASSWORD }} 41 | 42 | - uses: TykTechnologies/actions/docker-build-push@main 43 | with: 44 | dockerfile: Dockerfile 45 | tags: ${{ github.ref_name }} 46 | platforms: linux/amd64,linux/arm64 47 | repository_name: redis-cluster-operator 48 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yaml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | 3 | # Switch to only manual triggering for now. 4 | on: 5 | workflow_dispatch: 6 | 7 | jobs: 8 | golangci: 9 | name: lint 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | - uses: actions/setup-go@v5 14 | with: 15 | go-version: '1.23' 16 | - name: golangci-lint 17 | uses: golangci/golangci-lint-action@v7 18 | with: 19 | version: v2.0 20 | -------------------------------------------------------------------------------- /.github/workflows/test-e2e.yaml: -------------------------------------------------------------------------------- 1 | name: E2E Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main # Automatically trigger workflow for PRs targeting the main branch 10 | workflow_dispatch: # Manual trigger available for any branch 11 | 12 | jobs: 13 | e2e: 14 | runs-on: ubuntu-latest 15 | env: 16 | IMG: tykio/redis-cluster-operator:v0.0.0-teste2e 17 | CRUD_IMG: tykio/drc-crud-test:v0.0.0-teste2e 18 | KIND_CLUSTER: e2e-test 19 | KIND_CLUSTER_VERSION: v1.31.6 20 | KIND_CLUSTER_IMAGE: kindest/node:v1.31.6@sha256:28b7cbb993dfe093c76641a0c95807637213c9109b761f1d422c2400e22b8e87 21 | 22 | steps: 23 | # common steps for both tests 24 | - name: Checkout repository 25 | uses: actions/checkout@v4 26 | 27 | - name: Set up Go 28 | uses: actions/setup-go@v3 29 | with: 30 | go-version: '1.23' 31 | 32 | - name: Install kubectl 33 | uses: azure/setup-kubectl@v3 34 | with: 35 | version: '${{ env.KIND_CLUSTER_VERSION }}' 36 | 37 | - name: Set up Kind cluster 38 | uses: engineerd/setup-kind@v0.6.2 39 | with: 40 | version: "v0.27.0" 41 | image: ${{ env.KIND_CLUSTER_IMAGE }} 42 | name: ${{ env.KIND_CLUSTER }} 43 | 44 | - name: Verify cluster is running 45 | run: | 46 | kubectl cluster-info 47 | kubectl get nodes 48 | 49 | # run the drc_operator e2e tests 50 | - name: Run Redis Cluster Operator E2E Tests 51 | run: | 52 | go test ./test/e2e/drc_operator -v -ginkgo.v 53 | 54 | # steps for drc_crud test 55 | - name: Load Docker image into Kind cluster 56 | run: | 57 | kind load docker-image ${{ env.IMG }} --name e2e-test 58 | 59 | - name: Install CRDs 60 | run: | 61 | make install 62 | 63 | - name: Deploy controller-manager (operator) 64 | run: | 65 | make deploy IMG=${{ env.IMG }} 66 | 67 | - name: Build Distributed Redis Cluster CRUD E2E Test Image 68 | run: | 69 | make docker-build-e2e IMG=${{ env.CRUD_IMG }} 70 | 71 | - name: Load Distributed Redis Cluster CRUD E2E Test Image to Kind cluster 72 | run: | 73 | kind load docker-image ${{ env.CRUD_IMG }} --name e2e-test 74 | 75 | - name: Wait for operator to become available 76 | run: | 77 | kubectl wait --for=condition=available --timeout=90s deployment/redis-cluster-operator-controller-manager --namespace redis-cluster-operator-system 78 | 79 | # Run the drc_crud tests 80 | - name: Run Redis Cluster CRUD E2E Tests and stream pod logs 81 | run: | 82 | #!/bin/bash 83 | set -euo pipefail 84 | 85 | # Apply the job definition. 86 | kubectl apply -f test/e2e/drc_crud/job.yaml 87 | 88 | # Configuration 89 | namespace="redis-cluster-operator-system" 90 | job_label="job-name=drc-crud-e2e-job" 91 | pod_timeout="300s" # Timeout for pod readiness: 5 minutes 92 | job_timeout="1800s" # Timeout for job completion: 30 minutes 93 | 94 | echo "Waiting for pod with label '$job_label' in namespace '$namespace' to become ready..." 95 | 96 | if kubectl wait --for=condition=ready pod -n "$namespace" -l "$job_label" --timeout="$pod_timeout"; then 97 | pod=$(kubectl get pods -n "$namespace" -l "$job_label" -o jsonpath="{.items[0].metadata.name}") 98 | echo "Pod '$pod' is in the Ready state." 99 | else 100 | echo "Error: Timeout reached while waiting for the pod with label '$job_label' to become ready." 101 | exit 1 102 | fi 103 | 104 | echo "Streaming logs from pod '$pod'..." 105 | # It is generally safe, but consider if you need logs captured synchronously. 106 | kubectl logs --namespace "$namespace" -f "$pod" & 107 | 108 | echo "Waiting for job 'drc-crud-e2e-job' to complete..." 109 | if kubectl wait --for=condition=complete job/drc-crud-e2e-job -n "$namespace" --timeout="$job_timeout"; then 110 | echo "Job completed successfully." 111 | else 112 | echo "Error: The job did not complete successfully within the timeout period." 113 | exit 1 114 | fi 115 | shell: bash -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Temporary Build Files 2 | build/_output 3 | build/_test 4 | # Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode 5 | ### Emacs ### 6 | # -*- mode: gitignore; -*- 7 | *~ 8 | \#*\# 9 | /.emacs.desktop 10 | /.emacs.desktop.lock 11 | *.elc 12 | auto-save-list 13 | tramp 14 | .\#* 15 | # Org-mode 16 | .org-id-locations 17 | *_archive 18 | # flymake-mode 19 | *_flymake.* 20 | # eshell files 21 | /eshell/history 22 | /eshell/lastdir 23 | # elpa packages 24 | /elpa/ 25 | # reftex files 26 | *.rel 27 | # AUCTeX auto folder 28 | /auto/ 29 | # cask packages 30 | .cask/ 31 | dist/ 32 | # Flycheck 33 | flycheck_*.el 34 | # server auth directory 35 | /server/ 36 | # projectiles files 37 | .projectile 38 | projectile-bookmarks.eld 39 | # directory configuration 40 | .dir-locals.el 41 | # saveplace 42 | places 43 | # url cache 44 | url/cache/ 45 | # cedet 46 | ede-projects.el 47 | # smex 48 | smex-items 49 | # company-statistics 50 | company-statistics-cache.el 51 | # anaconda-mode 52 | anaconda-mode/ 53 | ### Go ### 54 | # Binaries for programs and plugins 55 | *.exe 56 | *.exe~ 57 | *.dll 58 | *.so 59 | *.dylib 60 | # Test binary, build with 'go test -c' 61 | *.test 62 | # Output of the go coverage tool, specifically when used with LiteIDE 63 | *.out 64 | ### Vim ### 65 | # swap 66 | .sw[a-p] 67 | .*.sw[a-p] 68 | # session 69 | Session.vim 70 | # temporary 71 | .netrwhist 72 | # auto-generated tag files 73 | tags 74 | ### VisualStudioCode ### 75 | .vscode/* 76 | .history 77 | # End of https://www.gitignore.io/api/go,vim,emacs,visualstudiocode 78 | .idea/* 79 | vendor/* 80 | /main 81 | /Dockerfile-withvendor 82 | Dockerfile-TZSH 83 | skaffold.yaml 84 | 85 | bin/ 86 | 87 | coverage.txt 88 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | run: 4 | timeout: 5m 5 | allow-parallel-runners: true 6 | 7 | issues: 8 | # don't skip warning about doc comments 9 | # don't exclude the default set of lint 10 | exclude-use-default: false 11 | # restore some of the defaults 12 | # (fill in the rest as needed) 13 | exclude-rules: 14 | - path: "api/*" 15 | linters: 16 | - lll 17 | - path: "pkg/*" 18 | linters: 19 | - dupl 20 | - lll 21 | 22 | linters: 23 | disable-all: true 24 | enable: 25 | - dupl 26 | - errcheck 27 | - ginkgolinter 28 | - goconst 29 | - gocyclo 30 | - govet 31 | - ineffassign 32 | - lll 33 | - misspell 34 | - nakedret 35 | - prealloc 36 | - revive 37 | - staticcheck 38 | - unconvert 39 | - unparam 40 | - unused 41 | 42 | linters-settings: 43 | revive: 44 | rules: 45 | - name: comment-spacings 46 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @TykTechnologies/tyk-cloud-sre @TykTechnologies/cloud-squad-be 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.23 AS builder 3 | ARG TARGETOS 4 | ARG TARGETARCH 5 | 6 | WORKDIR /workspace 7 | # Copy the Go Modules manifests 8 | COPY go.mod go.mod 9 | COPY go.sum go.sum 10 | # cache deps before building and copying source so that we don't need to re-download as much 11 | # and so that source changes don't invalidate our downloaded layer 12 | RUN go mod download 13 | 14 | # Copy the go source 15 | COPY cmd/main.go cmd/main.go 16 | COPY api/ api/ 17 | COPY internal/ internal/ 18 | 19 | # Build 20 | # the GOARCH has not a default value to allow the binary be built according to the host where the command 21 | # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO 22 | # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, 23 | # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. 24 | RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go 25 | 26 | # Use distroless as minimal base image to package the manager binary 27 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 28 | FROM gcr.io/distroless/static:nonroot 29 | WORKDIR / 30 | COPY --from=builder /workspace/manager . 31 | USER 65532:65532 32 | 33 | ENTRYPOINT ["/manager"] -------------------------------------------------------------------------------- /Dockerfile.e2e: -------------------------------------------------------------------------------- 1 | FROM golang:1.23 AS builder 2 | ARG TARGETOS 3 | ARG TARGETARCH 4 | 5 | # Set the working directory inside the container 6 | WORKDIR /workspace 7 | 8 | # Copy the Go Modules manifests 9 | COPY go.mod go.mod 10 | COPY go.sum go.sum 11 | # cache deps before building and copying source so that we don't need to re-download as much 12 | # and so that source changes don't invalidate our downloaded layer 13 | RUN go mod download 14 | 15 | # Copy the entire project (includes your test directory structure). 16 | COPY api/ api/ 17 | COPY internal/ internal/ 18 | COPY test/ test/ 19 | 20 | 21 | # Compile the end-to-end tests into a binary. 22 | # This command compiles tests in the ./test/e2e/ directory into an executable named "e2e-tests". 23 | RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go test -c -o /workspace/e2e-tests ./test/e2e/drc_crud/ 24 | 25 | FROM alpine:3.17 26 | 27 | WORKDIR / 28 | # Install CA certificates if your tests need to make HTTPS calls. 29 | RUN apk add --no-cache ca-certificates 30 | 31 | # Copy the test binary from the builder stage. 32 | COPY --from=builder /workspace/e2e-tests /e2e-tests 33 | 34 | # Set the entrypoint to run the test binary with the desired flags. 35 | ENTRYPOINT ["/e2e-tests", "-test.v", "-ginkgo.v"] 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2025. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | # Code generated by tool. DO NOT EDIT. 2 | # This file is used to track the info used to scaffold your project 3 | # and allow the plugins properly work. 4 | # More info: https://book.kubebuilder.io/reference/project-config.html 5 | domain: kun 6 | layout: 7 | - go.kubebuilder.io/v4 8 | plugins: 9 | manifests.sdk.operatorframework.io/v2: {} 10 | scorecard.sdk.operatorframework.io/v2: {} 11 | projectName: redis-cluster-operator-old 12 | repo: github.com/TykTechnologies/redis-cluster-operator 13 | resources: 14 | - api: 15 | crdVersion: v1 16 | namespaced: true 17 | controller: true 18 | domain: kun 19 | group: redis 20 | kind: DistributedRedisCluster 21 | path: github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1 22 | version: v1alpha1 23 | - api: 24 | crdVersion: v1 25 | namespaced: true 26 | controller: true 27 | domain: kun 28 | group: redis 29 | kind: RedisClusterBackup 30 | path: github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1 31 | version: v1alpha1 32 | - api: 33 | crdVersion: v1 34 | namespaced: true 35 | controller: true 36 | domain: kun 37 | group: redis 38 | kind: RedisClusterCleanup 39 | path: github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1 40 | version: v1alpha1 41 | version: "3" 42 | -------------------------------------------------------------------------------- /api/v1alpha1/constants.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | type StorageType string 4 | 5 | const ( 6 | PersistentClaim StorageType = "persistent-claim" 7 | Ephemeral StorageType = "ephemeral" 8 | ) 9 | 10 | const ( 11 | OperatorName = "redis-cluster-operator" 12 | LabelManagedByKey = "managed-by" 13 | LabelNameKey = "distributed-redis-cluster" 14 | StatefulSetLabel = "statefulSet" 15 | PasswordENV = "REDIS_PASSWORD" 16 | ) 17 | 18 | // RedisRole RedisCluster Node Role type 19 | type RedisRole string 20 | 21 | const ( 22 | // RedisClusterNodeRoleMaster RedisCluster Master node role 23 | RedisClusterNodeRoleMaster RedisRole = "Master" 24 | // RedisClusterNodeRoleSlave RedisCluster Master node role 25 | RedisClusterNodeRoleSlave RedisRole = "Slave" 26 | // RedisClusterNodeRoleNone None node role 27 | RedisClusterNodeRoleNone RedisRole = "None" 28 | ) 29 | 30 | // ClusterStatus Redis Cluster status 31 | type ClusterStatus string 32 | 33 | const ( 34 | // ClusterStatusOK ClusterStatus OK 35 | ClusterStatusOK ClusterStatus = "Healthy" 36 | // ClusterStatusKO ClusterStatus KO 37 | ClusterStatusKO ClusterStatus = "Failed" 38 | // ClusterStatusCreating ClusterStatus Creating 39 | ClusterStatusCreating = "Creating" 40 | // ClusterStatusScaling ClusterStatus Scaling 41 | ClusterStatusScaling ClusterStatus = "Scaling" 42 | // ClusterStatusCalculatingRebalancing ClusterStatus Rebalancing 43 | ClusterStatusCalculatingRebalancing ClusterStatus = "Calculating Rebalancing" 44 | // ClusterStatusRebalancing ClusterStatus Rebalancing 45 | ClusterStatusRebalancing ClusterStatus = "Rebalancing" 46 | // ClusterStatusRollingUpdate ClusterStatus RollingUpdate 47 | ClusterStatusRollingUpdate ClusterStatus = "RollingUpdate" 48 | // ClusterStatusResetPassword ClusterStatus ResetPassword 49 | ClusterStatusResetPassword ClusterStatus = "ResetPassword" 50 | ) 51 | 52 | // NodesPlacementInfo Redis Nodes placement mode information 53 | type NodesPlacementInfo string 54 | 55 | const ( 56 | // NodesPlacementInfoBestEffort the cluster nodes placement is in best effort, 57 | // it means you can have 2 masters (or more) on the same VM. 58 | NodesPlacementInfoBestEffort NodesPlacementInfo = "BestEffort" 59 | // NodesPlacementInfoOptimal the cluster nodes placement is optimal, 60 | // it means on master by VM 61 | NodesPlacementInfoOptimal NodesPlacementInfo = "Optimal" 62 | ) 63 | 64 | type RestorePhase string 65 | 66 | const ( 67 | // RestorePhaseRunning used for Restore that are currently running. 68 | RestorePhaseRunning RestorePhase = "Running" 69 | // RestorePhaseRestart used for Restore that are restart master nodes. 70 | RestorePhaseRestart RestorePhase = "Restart" 71 | // RestorePhaseSucceeded used for Restore that are Succeeded. 72 | RestorePhaseSucceeded RestorePhase = "Succeeded" 73 | ) 74 | 75 | const ( 76 | DatabaseNamePrefix = "redis" 77 | 78 | GenericKey = "redis.kun" 79 | 80 | LabelClusterName = GenericKey + "/name" 81 | 82 | BackupKey = ResourceSingularBackup + "." + GenericKey 83 | LabelBackupStatus = BackupKey + "/status" 84 | 85 | AnnotationJobType = GenericKey + "/job-type" 86 | 87 | JobTypeBackup = "backup" 88 | JobTypeRestore = "restore" 89 | 90 | PrometheusExporterPortNumber = 9100 91 | PrometheusExporterTelemetryPath = "/metrics" 92 | 93 | BackupDumpDir = "/data" 94 | UtilVolumeName = "util-volume" 95 | ) 96 | 97 | const ( 98 | DistributedRedisClusterKind = "DistributedRedisCluster" 99 | RedisClusterBackupKind = "RedisClusterBackup" 100 | RedisClusterCleanupKind = "RedisClusterCleanup" 101 | ) 102 | -------------------------------------------------------------------------------- /api/v1alpha1/default.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | 7 | "github.com/go-logr/logr" 8 | v1 "k8s.io/api/core/v1" 9 | "k8s.io/apimachinery/pkg/api/resource" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/runtime/schema" 12 | ) 13 | 14 | const ( 15 | minMasterSize = 3 16 | minClusterReplicas = 1 17 | defaultRedisImage = "redis:5.0.4-alpine" 18 | defaultMonitorImage = "oliver006/redis_exporter:latest" 19 | ) 20 | 21 | func (in *DistributedRedisCluster) DefaultSpec(log logr.Logger) bool { 22 | update := false 23 | if in.Spec.MasterSize < minMasterSize { 24 | in.Spec.MasterSize = minMasterSize 25 | update = true 26 | } 27 | 28 | if in.Spec.Image == "" { 29 | in.Spec.Image = defaultRedisImage 30 | update = true 31 | } 32 | 33 | if in.Spec.ServiceName == "" { 34 | in.Spec.ServiceName = in.Name 35 | update = true 36 | } 37 | 38 | if in.Spec.Resources == nil || in.Spec.Resources.Size() == 0 { 39 | in.Spec.Resources = defaultResource() 40 | update = true 41 | } 42 | 43 | mon := in.Spec.Monitor 44 | if mon != nil { 45 | if mon.Image == "" { 46 | mon.Image = defaultMonitorImage 47 | update = true 48 | } 49 | 50 | if mon.Prometheus == nil { 51 | mon.Prometheus = &PrometheusSpec{} 52 | update = true 53 | } 54 | if mon.Prometheus.Port == 0 { 55 | mon.Prometheus.Port = PrometheusExporterPortNumber 56 | update = true 57 | } 58 | if in.Spec.Annotations == nil { 59 | in.Spec.Annotations = make(map[string]string) 60 | update = true 61 | } 62 | 63 | in.Spec.Annotations["prometheus.io/scrape"] = "true" 64 | in.Spec.Annotations["prometheus.io/path"] = PrometheusExporterTelemetryPath 65 | in.Spec.Annotations["prometheus.io/port"] = fmt.Sprintf("%d", mon.Prometheus.Port) 66 | } 67 | return update 68 | } 69 | 70 | func (in *DistributedRedisCluster) IsRestoreFromBackup() bool { 71 | initSpec := in.Spec.Init 72 | if initSpec != nil && initSpec.BackupSource != nil { 73 | return true 74 | } 75 | return false 76 | } 77 | 78 | func (in *DistributedRedisCluster) IsRestored() bool { 79 | return in.Status.Restore.Phase == RestorePhaseSucceeded 80 | } 81 | 82 | func (in *DistributedRedisCluster) ShouldInitRestorePhase() bool { 83 | return in.Status.Restore.Phase == "" 84 | } 85 | 86 | func (in *DistributedRedisCluster) IsRestoreRunning() bool { 87 | return in.Status.Restore.Phase == RestorePhaseRunning 88 | } 89 | 90 | func (in *DistributedRedisCluster) IsRestoreRestarting() bool { 91 | return in.Status.Restore.Phase == RestorePhaseRestart 92 | } 93 | 94 | func defaultResource() *v1.ResourceRequirements { 95 | return &v1.ResourceRequirements{ 96 | Requests: v1.ResourceList{ 97 | v1.ResourceCPU: resource.MustParse("200m"), 98 | v1.ResourceMemory: resource.MustParse("2Gi"), 99 | }, 100 | Limits: v1.ResourceList{ 101 | v1.ResourceCPU: resource.MustParse("1000m"), 102 | v1.ResourceMemory: resource.MustParse("4Gi"), 103 | }, 104 | } 105 | } 106 | 107 | func DefaultOwnerReferences(cluster *DistributedRedisCluster) []metav1.OwnerReference { 108 | return []metav1.OwnerReference{ 109 | *metav1.NewControllerRef(cluster, schema.GroupVersionKind{ 110 | Group: GroupVersion.Group, 111 | Version: GroupVersion.Version, 112 | Kind: DistributedRedisClusterKind, 113 | }), 114 | } 115 | } 116 | 117 | func (in *RedisClusterBackup) Validate() error { 118 | clusterName := in.Spec.RedisClusterName 119 | if clusterName == "" { 120 | return fmt.Errorf("bakcup [RedisClusterName] is missing") 121 | } 122 | // BucketName can't be empty 123 | if in.Spec.S3 == nil && in.Spec.GCS == nil && in.Spec.Azure == nil && in.Spec.Swift == nil && in.Spec.Local == nil { 124 | return fmt.Errorf("no storage provider is configured") 125 | } 126 | 127 | if in.Spec.Azure != nil || in.Spec.Swift != nil { 128 | if in.Spec.StorageSecretName == "" { 129 | return fmt.Errorf("bakcup [SecretName] is missing") 130 | } 131 | } 132 | return nil 133 | } 134 | 135 | func (in *RedisClusterBackup) RemotePath() (string, error) { 136 | spec := in.Spec.Backend 137 | timePrefix := in.Status.StartTime.Format("20060102150405") 138 | if spec.S3 != nil { 139 | return filepath.Join(spec.S3.Prefix, DatabaseNamePrefix, in.Namespace, in.Spec.RedisClusterName, timePrefix), nil 140 | } else if spec.GCS != nil { 141 | return filepath.Join(spec.GCS.Prefix, DatabaseNamePrefix, in.Namespace, in.Spec.RedisClusterName, timePrefix), nil 142 | } else if spec.Azure != nil { 143 | return filepath.Join(spec.Azure.Prefix, DatabaseNamePrefix, in.Namespace, in.Spec.RedisClusterName, timePrefix), nil 144 | } else if spec.Local != nil { 145 | return filepath.Join(DatabaseNamePrefix, in.Namespace, in.Spec.RedisClusterName, timePrefix), nil 146 | } else if spec.Swift != nil { 147 | return filepath.Join(spec.Swift.Prefix, DatabaseNamePrefix, in.Namespace, in.Spec.RedisClusterName, timePrefix), nil 148 | } 149 | return "", fmt.Errorf("no storage provider is configured") 150 | } 151 | 152 | func (in *RedisClusterBackup) RCloneSecretName() string { 153 | return fmt.Sprintf("rcloneconfig-%v", in.Name) 154 | } 155 | 156 | func (in *RedisClusterBackup) JobName() string { 157 | return fmt.Sprintf("redisbackup-%v", in.Name) 158 | } 159 | 160 | func (in *RedisClusterBackup) IsRefLocalPVC() bool { 161 | return in.Spec.Local != nil && in.Spec.Local.PersistentVolumeClaim != nil 162 | } 163 | -------------------------------------------------------------------------------- /api/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2025. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1alpha1 contains API Schema definitions for the redis v1alpha1 API group 18 | // +kubebuilder:object:generate=true 19 | // +groupName=redis.kun 20 | package v1alpha1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "redis.kun", Version: "v1alpha1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /api/v1alpha1/redisclustercleanup_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2025. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! 24 | // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. 25 | 26 | // RedisClusterCleanupSpec defines the desired state of RedisClusterCleanup 27 | type RedisClusterCleanupSpec struct { 28 | // Schedule is a cron expression to run the cleanup job. 29 | // +kubebuilder:validation:Pattern=`^(\S+\s+){4}\S+$` 30 | Schedule string `json:"schedule"` 31 | 32 | // +kubebuilder:default:=false 33 | Suspend bool `json:"suspend,omitempty"` 34 | 35 | // ExpiredThreshold defines the minimum number of expired keys that triggers a cleanup. 36 | // +kubebuilder:default:=200 37 | ExpiredThreshold int `json:"expiredThreshold,omitempty"` 38 | 39 | // ExpiredThreshold defines the minimum number of expired keys that triggers a cleanup. 40 | // +kubebuilder:default:=200 41 | ScanBatchSize int `json:"scanBatchSize,omitempty"` 42 | 43 | // +kubebuilder:validation:Required 44 | // +kubebuilder:validation:MinItems=1 45 | Namespaces []string `json:"namespaces"` 46 | 47 | // KeyPatterns holds one or more patterns for SCAN operations. 48 | // For example, ["apikey-*", "session-*"] 49 | KeyPatterns []string `json:"keyPatterns,omitempty"` 50 | 51 | // ExpirationRegexes holds one or more regexes to extract the expiration value. 52 | // For example: ["\"expires\":\\s*(\\d+)"] 53 | ExpirationRegexes []string `json:"expirationRegexes,omitempty"` 54 | 55 | // SkipPatterns holds substrings or patterns that if found in a key's value will skip deletion. 56 | // For example, ["TykJWTSessionID"] 57 | SkipPatterns []string `json:"skipPatterns,omitempty"` 58 | } 59 | 60 | // RedisClusterCleanupStatus defines the observed state of RedisClusterCleanup 61 | type RedisClusterCleanupStatus struct { 62 | // LastScheduleTime is the last time the CronJob was scheduled. 63 | LastScheduleTime *metav1.Time `json:"lastScheduleTime,omitempty"` 64 | // LastSuccessfulTime is the last time the CronJob completed successfully. 65 | LastSuccessfulTime *metav1.Time `json:"lastSuccessfulTime,omitempty"` 66 | 67 | Succeed int64 `json:"Succeed,omitempty"` 68 | } 69 | 70 | // +kubebuilder:resource:shortName="drcc" 71 | // +kubebuilder:object:root=true 72 | // +kubebuilder:subresource:status 73 | // +kubebuilder:printcolumn:name="Schedule",type="string",JSONPath=".spec.schedule",description="Cleanup schedule" 74 | // +kubebuilder:printcolumn:name="Suspend",type="boolean",JSONPath=".spec.suspend",description="Whether the Cleaner is currently suspended (True/False)." 75 | // +kubebuilder:printcolumn:name="LastSuccessfulTime",type="date",JSONPath=".status.lastSuccessfulTime",description="The last time the Cleaner completed successfully" 76 | 77 | // RedisClusterCleanup is the Schema for the redisclustercleanups API 78 | type RedisClusterCleanup struct { 79 | metav1.TypeMeta `json:",inline"` 80 | metav1.ObjectMeta `json:"metadata,omitempty"` 81 | 82 | Spec RedisClusterCleanupSpec `json:"spec,omitempty"` 83 | Status RedisClusterCleanupStatus `json:"status,omitempty"` 84 | } 85 | 86 | // +kubebuilder:object:root=true 87 | 88 | // RedisClusterCleanupList contains a list of RedisClusterCleanup 89 | type RedisClusterCleanupList struct { 90 | metav1.TypeMeta `json:",inline"` 91 | metav1.ListMeta `json:"metadata,omitempty"` 92 | Items []RedisClusterCleanup `json:"items"` 93 | } 94 | 95 | func init() { 96 | SchemeBuilder.Register(&RedisClusterCleanup{}, &RedisClusterCleanupList{}) 97 | } 98 | -------------------------------------------------------------------------------- /config/crd/bases/redis.kun_redisclustercleanups.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | controller-gen.kubebuilder.io/version: v0.16.1 7 | name: redisclustercleanups.redis.kun 8 | spec: 9 | group: redis.kun 10 | names: 11 | kind: RedisClusterCleanup 12 | listKind: RedisClusterCleanupList 13 | plural: redisclustercleanups 14 | shortNames: 15 | - drcc 16 | singular: redisclustercleanup 17 | scope: Namespaced 18 | versions: 19 | - additionalPrinterColumns: 20 | - description: Cleanup schedule 21 | jsonPath: .spec.schedule 22 | name: Schedule 23 | type: string 24 | - description: Whether the Cleaner is currently suspended (True/False). 25 | jsonPath: .spec.suspend 26 | name: Suspend 27 | type: boolean 28 | - description: The last time the Cleaner completed successfully 29 | jsonPath: .status.lastSuccessfulTime 30 | name: LastSuccessfulTime 31 | type: date 32 | name: v1alpha1 33 | schema: 34 | openAPIV3Schema: 35 | description: RedisClusterCleanup is the Schema for the redisclustercleanups 36 | API 37 | properties: 38 | apiVersion: 39 | description: |- 40 | APIVersion defines the versioned schema of this representation of an object. 41 | Servers should convert recognized schemas to the latest internal value, and 42 | may reject unrecognized values. 43 | More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources 44 | type: string 45 | kind: 46 | description: |- 47 | Kind is a string value representing the REST resource this object represents. 48 | Servers may infer this from the endpoint the client submits requests to. 49 | Cannot be updated. 50 | In CamelCase. 51 | More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds 52 | type: string 53 | metadata: 54 | type: object 55 | spec: 56 | description: RedisClusterCleanupSpec defines the desired state of RedisClusterCleanup 57 | properties: 58 | expirationRegexes: 59 | description: |- 60 | ExpirationRegexes holds one or more regexes to extract the expiration value. 61 | For example: ["\"expires\":\\s*(\\d+)"] 62 | items: 63 | type: string 64 | type: array 65 | expiredThreshold: 66 | default: 200 67 | description: ExpiredThreshold defines the minimum number of expired 68 | keys that triggers a cleanup. 69 | type: integer 70 | keyPatterns: 71 | description: |- 72 | KeyPatterns holds one or more patterns for SCAN operations. 73 | For example, ["apikey-*", "session-*"] 74 | items: 75 | type: string 76 | type: array 77 | namespaces: 78 | items: 79 | type: string 80 | minItems: 1 81 | type: array 82 | scanBatchSize: 83 | default: 200 84 | description: ExpiredThreshold defines the minimum number of expired 85 | keys that triggers a cleanup. 86 | type: integer 87 | schedule: 88 | description: Schedule is a cron expression to run the cleanup job. 89 | pattern: ^(\S+\s+){4}\S+$ 90 | type: string 91 | skipPatterns: 92 | description: |- 93 | SkipPatterns holds substrings or patterns that if found in a key's value will skip deletion. 94 | For example, ["TykJWTSessionID"] 95 | items: 96 | type: string 97 | type: array 98 | suspend: 99 | default: false 100 | type: boolean 101 | required: 102 | - namespaces 103 | - schedule 104 | type: object 105 | status: 106 | description: RedisClusterCleanupStatus defines the observed state of RedisClusterCleanup 107 | properties: 108 | Succeed: 109 | format: int64 110 | type: integer 111 | lastScheduleTime: 112 | description: LastScheduleTime is the last time the CronJob was scheduled. 113 | format: date-time 114 | type: string 115 | lastSuccessfulTime: 116 | description: LastSuccessfulTime is the last time the CronJob completed 117 | successfully. 118 | format: date-time 119 | type: string 120 | type: object 121 | type: object 122 | served: true 123 | storage: true 124 | subresources: 125 | status: {} 126 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | - bases/redis.kun_distributedredisclusters.yaml 6 | - bases/redis.kun_redisclusterbackups.yaml 7 | - bases/redis.kun_redisclustercleanups.yaml 8 | # +kubebuilder:scaffold:crdkustomizeresource 9 | 10 | patches: 11 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 12 | # patches here are for enabling the conversion webhook for each CRD 13 | # +kubebuilder:scaffold:crdkustomizewebhookpatch 14 | 15 | # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. 16 | # patches here are for enabling the CA injection for each CRD 17 | #- path: patches/cainjection_in_distributedredisclusters.yaml 18 | #- path: patches/cainjection_in_redisclusterbackups.yaml 19 | #- path: patches/cainjection_in_redisclustercleanups.yaml 20 | # +kubebuilder:scaffold:crdkustomizecainjectionpatch 21 | 22 | # [WEBHOOK] To enable webhook, uncomment the following section 23 | # the following config is for teaching kustomize how to do kustomization for CRDs. 24 | 25 | #configurations: 26 | #- kustomizeconfig.yaml 27 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | version: v1 8 | group: apiextensions.k8s.io 9 | path: spec/conversion/webhook/clientConfig/service/name 10 | 11 | namespace: 12 | - kind: CustomResourceDefinition 13 | version: v1 14 | group: apiextensions.k8s.io 15 | path: spec/conversion/webhook/clientConfig/service/namespace 16 | create: false 17 | 18 | varReference: 19 | - path: metadata/annotations 20 | -------------------------------------------------------------------------------- /config/default/manager_metrics_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch adds the args to allow exposing the metrics endpoint using HTTPS 2 | - op: add 3 | path: /spec/template/spec/containers/0/args/0 4 | value: --metrics-bind-address=:8443 5 | -------------------------------------------------------------------------------- /config/default/metrics_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | app.kubernetes.io/name: redis-cluster-operator-old 7 | app.kubernetes.io/managed-by: kustomize 8 | name: controller-manager-metrics-service 9 | namespace: system 10 | spec: 11 | ports: 12 | - name: https 13 | port: 8443 14 | protocol: TCP 15 | targetPort: 8443 16 | selector: 17 | control-plane: controller-manager 18 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | apiVersion: kustomize.config.k8s.io/v1beta1 4 | kind: Kustomization 5 | images: 6 | - name: controller 7 | newName: tykio/redis-cluster-operator 8 | newTag: v0.1.0 9 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | labels: 6 | control-plane: controller-manager 7 | app.kubernetes.io/name: redis-cluster-operator 8 | app.kubernetes.io/managed-by: kustomize 9 | name: system 10 | --- 11 | apiVersion: v1 12 | kind: ConfigMap 13 | metadata: 14 | name: redisconf 15 | data: 16 | redis.conf: |- 17 | rename-command CONFIG lni07z1p 18 | --- 19 | apiVersion: apps/v1 20 | kind: Deployment 21 | metadata: 22 | name: controller-manager 23 | namespace: system 24 | labels: 25 | control-plane: controller-manager 26 | app.kubernetes.io/name: redis-cluster-operator 27 | app.kubernetes.io/managed-by: kustomize 28 | spec: 29 | selector: 30 | matchLabels: 31 | control-plane: controller-manager 32 | replicas: 1 33 | template: 34 | metadata: 35 | annotations: 36 | kubectl.kubernetes.io/default-container: manager 37 | labels: 38 | control-plane: controller-manager 39 | spec: 40 | # according to the platforms which are supported by your solution. 41 | # It is considered best practice to support multiple architectures. You can 42 | # build your manager image using the makefile target docker-buildx. 43 | # affinity: 44 | # nodeAffinity: 45 | # requiredDuringSchedulingIgnoredDuringExecution: 46 | # nodeSelectorTerms: 47 | # - matchExpressions: 48 | # - key: kubernetes.io/arch 49 | # operator: In 50 | # values: 51 | # - amd64 52 | # - arm64 53 | # - ppc64le 54 | # - s390x 55 | # - key: kubernetes.io/os 56 | # operator: In 57 | # values: 58 | # - linux 59 | securityContext: 60 | runAsNonRoot: true 61 | # it is recommended to ensure that all your Pods/Containers are restrictive. 62 | # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted 63 | # Please uncomment the following code if your project does NOT have to work on old Kubernetes 64 | # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). 65 | seccompProfile: 66 | type: RuntimeDefault 67 | containers: 68 | - command: 69 | - /manager 70 | args: 71 | - --leader-elect 72 | - --health-probe-bind-address=:8081 73 | - --metrics-bind-address=:8443 74 | - --metrics-secure=false 75 | - --rename-command-path=/etc/redisconf 76 | - --rename-command-file=redis.conf 77 | image: controller:latest 78 | name: manager 79 | securityContext: 80 | allowPrivilegeEscalation: false 81 | capabilities: 82 | drop: 83 | - "ALL" 84 | readOnlyRootFilesystem: true 85 | ports: 86 | - name: http-metrics 87 | containerPort: 8443 88 | protocol: TCP 89 | livenessProbe: 90 | httpGet: 91 | path: /healthz 92 | port: 8081 93 | initialDelaySeconds: 15 94 | periodSeconds: 20 95 | readinessProbe: 96 | httpGet: 97 | path: /readyz 98 | port: 8081 99 | initialDelaySeconds: 5 100 | periodSeconds: 10 101 | resources: 102 | limits: 103 | cpu: 500m 104 | memory: 128Mi 105 | requests: 106 | cpu: 10m 107 | memory: 64Mi 108 | volumeMounts: 109 | - name: redisconf 110 | mountPath: /etc/redisconf 111 | volumes: 112 | - name: redisconf 113 | configMap: 114 | name: redisconf 115 | serviceAccountName: controller-manager 116 | terminationGracePeriodSeconds: 10 117 | -------------------------------------------------------------------------------- /config/manifests/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # These resources constitute the fully configured set of manifests 2 | # used to generate the 'manifests/' directory in a bundle. 3 | resources: 4 | - bases/redis-cluster-operator-old.clusterserviceversion.yaml 5 | - ../default 6 | - ../samples 7 | - ../scorecard 8 | 9 | # [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. 10 | # Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. 11 | # These patches remove the unnecessary "cert" volume and its manager container volumeMount. 12 | #patches: 13 | #- target: 14 | # group: apps 15 | # version: v1 16 | # kind: Deployment 17 | # name: controller-manager 18 | # namespace: system 19 | # patch: |- 20 | # # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. 21 | # # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. 22 | # - op: remove 23 | 24 | # path: /spec/template/spec/containers/0/volumeMounts/0 25 | # # Remove the "cert" volume, since OLM will create and mount a set of certs. 26 | # # Update the indices in this path if adding or removing volumes in the manager's Deployment. 27 | # - op: remove 28 | # path: /spec/template/spec/volumes/0 29 | -------------------------------------------------------------------------------- /config/network-policy/allow-metrics-traffic.yaml: -------------------------------------------------------------------------------- 1 | # This NetworkPolicy allows ingress traffic 2 | # with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those 3 | # namespaces are able to gathering data from the metrics endpoint. 4 | apiVersion: networking.k8s.io/v1 5 | kind: NetworkPolicy 6 | metadata: 7 | labels: 8 | app.kubernetes.io/name: redis-cluster-operator-old 9 | app.kubernetes.io/managed-by: kustomize 10 | name: allow-metrics-traffic 11 | namespace: system 12 | spec: 13 | podSelector: 14 | matchLabels: 15 | control-plane: controller-manager 16 | policyTypes: 17 | - Ingress 18 | ingress: 19 | # This allows ingress traffic from any namespace with the label metrics: enabled 20 | - from: 21 | - namespaceSelector: 22 | matchLabels: 23 | metrics: enabled # Only from namespaces with this label 24 | ports: 25 | - port: 8443 26 | protocol: TCP 27 | -------------------------------------------------------------------------------- /config/network-policy/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - allow-metrics-traffic.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | # Prometheus Monitor Service (Metrics) 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | labels: 6 | control-plane: controller-manager 7 | app.kubernetes.io/name: redis-cluster-operator-old 8 | app.kubernetes.io/managed-by: kustomize 9 | name: controller-manager-metrics-monitor 10 | namespace: system 11 | spec: 12 | endpoints: 13 | - path: /metrics 14 | port: https # Ensure this is the name of the port that exposes HTTPS metrics 15 | scheme: https 16 | bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token 17 | tlsConfig: 18 | # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables 19 | # certificate verification. This poses a significant security risk by making the system vulnerable to 20 | # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between 21 | # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, 22 | # compromising the integrity and confidentiality of the information. 23 | # Please use the following options for secure configurations: 24 | # caFile: /etc/metrics-certs/ca.crt 25 | # certFile: /etc/metrics-certs/tls.crt 26 | # keyFile: /etc/metrics-certs/tls.key 27 | insecureSkipVerify: true 28 | selector: 29 | matchLabels: 30 | control-plane: controller-manager 31 | -------------------------------------------------------------------------------- /config/rbac/distributedrediscluster_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit distributedredisclusters. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: redis-cluster-operator-old 7 | app.kubernetes.io/managed-by: kustomize 8 | name: distributedrediscluster-editor-role 9 | rules: 10 | - apiGroups: 11 | - redis.kun 12 | resources: 13 | - distributedredisclusters 14 | verbs: 15 | - create 16 | - delete 17 | - get 18 | - list 19 | - patch 20 | - update 21 | - watch 22 | - apiGroups: 23 | - redis.kun 24 | resources: 25 | - distributedredisclusters/status 26 | verbs: 27 | - get 28 | -------------------------------------------------------------------------------- /config/rbac/distributedrediscluster_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view distributedredisclusters. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: redis-cluster-operator-old 7 | app.kubernetes.io/managed-by: kustomize 8 | name: distributedrediscluster-viewer-role 9 | rules: 10 | - apiGroups: 11 | - redis.kun 12 | resources: 13 | - distributedredisclusters 14 | verbs: 15 | - get 16 | - list 17 | - watch 18 | - apiGroups: 19 | - redis.kun 20 | resources: 21 | - distributedredisclusters/status 22 | verbs: 23 | - get 24 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | # All RBAC will be applied under this service account in 3 | # the deployment namespace. You may comment out this resource 4 | # if your manager will use a service account that exists at 5 | # runtime. Be sure to update RoleBinding and ClusterRoleBinding 6 | # subjects if changing service account names. 7 | - service_account.yaml 8 | - role.yaml 9 | - role_binding.yaml 10 | - leader_election_role.yaml 11 | - leader_election_role_binding.yaml 12 | # The following RBAC configurations are used to protect 13 | # the metrics endpoint with authn/authz. These configurations 14 | # ensure that only authorized users and service accounts 15 | # can access the metrics endpoint. Comment the following 16 | # permissions if you want to disable this protection. 17 | # More info: https://book.kubebuilder.io/reference/metrics.html 18 | - metrics_auth_role.yaml 19 | - metrics_auth_role_binding.yaml 20 | - metrics_reader_role.yaml 21 | # For each CRD, "Editor" and "Viewer" roles are scaffolded by 22 | # default, aiding admins in cluster management. Those roles are 23 | # not used by the Project itself. You can comment the following lines 24 | # if you do not want those helpers be installed with your Project. 25 | - redisclustercleanup_editor_role.yaml 26 | - redisclustercleanup_viewer_role.yaml 27 | - redisclusterbackup_editor_role.yaml 28 | - redisclusterbackup_viewer_role.yaml 29 | - distributedrediscluster_editor_role.yaml 30 | - distributedrediscluster_viewer_role.yaml 31 | 32 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: redis-cluster-operator-old 7 | app.kubernetes.io/managed-by: kustomize 8 | name: leader-election-role 9 | rules: 10 | - apiGroups: 11 | - "" 12 | resources: 13 | - configmaps 14 | verbs: 15 | - get 16 | - list 17 | - watch 18 | - create 19 | - update 20 | - patch 21 | - delete 22 | - apiGroups: 23 | - coordination.k8s.io 24 | resources: 25 | - leases 26 | verbs: 27 | - get 28 | - list 29 | - watch 30 | - create 31 | - update 32 | - patch 33 | - delete 34 | - apiGroups: 35 | - "" 36 | resources: 37 | - events 38 | verbs: 39 | - create 40 | - patch 41 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: redis-cluster-operator-old 6 | app.kubernetes.io/managed-by: kustomize 7 | name: leader-election-rolebinding 8 | roleRef: 9 | apiGroup: rbac.authorization.k8s.io 10 | kind: Role 11 | name: leader-election-role 12 | subjects: 13 | - kind: ServiceAccount 14 | name: controller-manager 15 | namespace: system 16 | -------------------------------------------------------------------------------- /config/rbac/metrics_auth_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-auth-role 5 | rules: 6 | - apiGroups: 7 | - authentication.k8s.io 8 | resources: 9 | - tokenreviews 10 | verbs: 11 | - create 12 | - apiGroups: 13 | - authorization.k8s.io 14 | resources: 15 | - subjectaccessreviews 16 | verbs: 17 | - create 18 | -------------------------------------------------------------------------------- /config/rbac/metrics_auth_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: metrics-auth-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: metrics-auth-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/metrics_reader_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: 7 | - "/metrics" 8 | verbs: 9 | - get 10 | -------------------------------------------------------------------------------- /config/rbac/redisclusterbackup_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit redisclusterbackups. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: redis-cluster-operator-old 7 | app.kubernetes.io/managed-by: kustomize 8 | name: redisclusterbackup-editor-role 9 | rules: 10 | - apiGroups: 11 | - redis.kun 12 | resources: 13 | - redisclusterbackups 14 | verbs: 15 | - create 16 | - delete 17 | - get 18 | - list 19 | - patch 20 | - update 21 | - watch 22 | - apiGroups: 23 | - redis.kun 24 | resources: 25 | - redisclusterbackups/status 26 | verbs: 27 | - get 28 | -------------------------------------------------------------------------------- /config/rbac/redisclusterbackup_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view redisclusterbackups. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: redis-cluster-operator-old 7 | app.kubernetes.io/managed-by: kustomize 8 | name: redisclusterbackup-viewer-role 9 | rules: 10 | - apiGroups: 11 | - redis.kun 12 | resources: 13 | - redisclusterbackups 14 | verbs: 15 | - get 16 | - list 17 | - watch 18 | - apiGroups: 19 | - redis.kun 20 | resources: 21 | - redisclusterbackups/status 22 | verbs: 23 | - get 24 | -------------------------------------------------------------------------------- /config/rbac/redisclustercleanup_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit redisclustercleanups. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: redis-cluster-operator-old 7 | app.kubernetes.io/managed-by: kustomize 8 | name: redisclustercleanup-editor-role 9 | rules: 10 | - apiGroups: 11 | - redis.kun 12 | resources: 13 | - redisclustercleanups 14 | verbs: 15 | - create 16 | - delete 17 | - get 18 | - list 19 | - patch 20 | - update 21 | - watch 22 | - apiGroups: 23 | - redis.kun 24 | resources: 25 | - redisclustercleanups/status 26 | verbs: 27 | - get 28 | -------------------------------------------------------------------------------- /config/rbac/redisclustercleanup_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view redisclustercleanups. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: redis-cluster-operator-old 7 | app.kubernetes.io/managed-by: kustomize 8 | name: redisclustercleanup-viewer-role 9 | rules: 10 | - apiGroups: 11 | - redis.kun 12 | resources: 13 | - redisclustercleanups 14 | verbs: 15 | - get 16 | - list 17 | - watch 18 | - apiGroups: 19 | - redis.kun 20 | resources: 21 | - redisclustercleanups/status 22 | verbs: 23 | - get 24 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: manager-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | - events 12 | - persistentvolumeclaims 13 | - pods/exec 14 | - secrets 15 | - services 16 | verbs: 17 | - create 18 | - delete 19 | - get 20 | - list 21 | - patch 22 | - update 23 | - watch 24 | - apiGroups: 25 | - "" 26 | resources: 27 | - endpoints 28 | - pods 29 | verbs: 30 | - delete 31 | - get 32 | - list 33 | - watch 34 | - apiGroups: 35 | - "" 36 | resources: 37 | - namespaces 38 | verbs: 39 | - get 40 | - list 41 | - watch 42 | - apiGroups: 43 | - apps 44 | resources: 45 | - deployments 46 | - replicasets 47 | - statefulsets 48 | verbs: 49 | - create 50 | - delete 51 | - get 52 | - list 53 | - patch 54 | - update 55 | - watch 56 | - apiGroups: 57 | - batch 58 | resources: 59 | - jobs 60 | verbs: 61 | - create 62 | - delete 63 | - get 64 | - list 65 | - patch 66 | - update 67 | - watch 68 | - apiGroups: 69 | - policy 70 | resources: 71 | - poddisruptionbudgets 72 | verbs: 73 | - create 74 | - delete 75 | - get 76 | - list 77 | - patch 78 | - update 79 | - watch 80 | - apiGroups: 81 | - redis.kun 82 | resources: 83 | - distributedredisclusters 84 | - redisclusterbackups 85 | - redisclustercleanups 86 | verbs: 87 | - create 88 | - delete 89 | - get 90 | - list 91 | - patch 92 | - update 93 | - watch 94 | - apiGroups: 95 | - redis.kun 96 | resources: 97 | - distributedredisclusters/finalizers 98 | - redisclusterbackups/finalizers 99 | - redisclustercleanups/finalizers 100 | verbs: 101 | - update 102 | - apiGroups: 103 | - redis.kun 104 | resources: 105 | - distributedredisclusters/status 106 | - redisclusterbackups/status 107 | - redisclustercleanups/status 108 | verbs: 109 | - get 110 | - patch 111 | - update 112 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: redis-cluster-operator-old 6 | app.kubernetes.io/managed-by: kustomize 7 | name: manager-rolebinding 8 | roleRef: 9 | apiGroup: rbac.authorization.k8s.io 10 | kind: ClusterRole 11 | name: manager-role 12 | subjects: 13 | - kind: ServiceAccount 14 | name: controller-manager 15 | namespace: system 16 | -------------------------------------------------------------------------------- /config/rbac/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: redis-cluster-operator-old 6 | app.kubernetes.io/managed-by: kustomize 7 | name: controller-manager 8 | namespace: system 9 | -------------------------------------------------------------------------------- /config/samples/example/backup-restore/redisclusterbackup_cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | AWS_ACCESS_KEY_ID: dGVzdA== 4 | AWS_SECRET_ACCESS_KEY: dGVzdA== 5 | kind: Secret 6 | metadata: 7 | name: s3-secret 8 | type: Opaque 9 | --- 10 | apiVersion: redis.kun/v1alpha1 11 | kind: RedisClusterBackup 12 | metadata: 13 | annotations: 14 | # if your operator run as cluster-scoped, add this annotations 15 | redis.kun/scope: cluster-scoped 16 | name: example-redisclusterbackup 17 | spec: 18 | image: redis-tools:5.0.4 19 | redisClusterName: example-distributedrediscluster 20 | storageSecretName: s3-secret 21 | # Replace this with the s3 info 22 | s3: 23 | endpoint: REPLACE_ENDPOINT 24 | bucket: REPLACE_BUCKET 25 | -------------------------------------------------------------------------------- /config/samples/example/backup-restore/redisclusterbackup_topvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: test-backup 5 | spec: 6 | accessModes: 7 | - ReadWriteMany 8 | resources: 9 | requests: 10 | storage: 10Gi 11 | storageClassName: {storageClassName} 12 | volumeMode: Filesystem 13 | --- 14 | 15 | apiVersion: redis.kun/v1alpha1 16 | kind: RedisClusterBackup 17 | metadata: 18 | name: example-redisclusterbackup 19 | annotations: 20 | redis.kun/scope: cluster-scoped 21 | spec: 22 | image: hub.ucloudadmin.com/uaek/redis-tools:5.0.4 23 | # on same namespace 24 | redisClusterName: test 25 | local: 26 | mountPath: /back 27 | persistentVolumeClaim: 28 | claimName: test-backup 29 | -------------------------------------------------------------------------------- /config/samples/example/backup-restore/restore.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: DistributedRedisCluster 3 | metadata: 4 | annotations: 5 | # if your operator run as cluster-scoped, add this annotations 6 | redis.kun/scope: cluster-scoped 7 | name: example-restore 8 | spec: 9 | init: 10 | backupSource: 11 | name: example-redisclusterbackup 12 | namespace: default 13 | -------------------------------------------------------------------------------- /config/samples/example/backup-restore/restore_frompvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: DistributedRedisCluster 3 | metadata: 4 | name: restore 5 | spec: 6 | clusterReplicas: 1 7 | config: 8 | appendfsync: everysec 9 | appendonly: "yes" 10 | auto-aof-rewrite-min-size: 64mb 11 | auto-aof-rewrite-percentage: "100" 12 | cluster-node-timeout: "5000" 13 | loglevel: verbose 14 | maxclients: "1000" 15 | maxmemory: "0" 16 | notify-keyspace-events: "" 17 | rdbcompression: "yes" 18 | save: 900 1 300 10 19 | stop-writes-on-bgsave-error: "yes" 20 | tcp-keepalive: "0" 21 | timeout: "0" 22 | image: redis:5.0.4-alpine 23 | masterSize: 3 24 | resources: 25 | limits: 26 | cpu: 400m 27 | memory: 300Mi 28 | requests: 29 | cpu: 400m 30 | memory: 300Mi 31 | storage: 32 | class: {storageClassName} 33 | size: 10Gi 34 | type: persistent-claim 35 | init: 36 | backupSource: 37 | name: example-redisclusterbackup 38 | namespace: default 39 | -------------------------------------------------------------------------------- /config/samples/example/custom-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: DistributedRedisCluster 3 | metadata: 4 | annotations: 5 | # if your operator run as cluster-scoped, add this annotations 6 | redis.kun/scope: cluster-scoped 7 | name: example-distributedrediscluster 8 | spec: 9 | image: redis:5.0.4-alpine 10 | masterSize: 3 11 | clusterReplicas: 1 12 | config: 13 | activerehashing: "yes" 14 | appendfsync: everysec 15 | appendonly: "yes" 16 | hash-max-ziplist-entries: "512" 17 | hash-max-ziplist-value: "64" 18 | hll-sparse-max-bytes: "3000" 19 | list-compress-depth: "0" 20 | maxmemory-policy: noeviction 21 | maxmemory-samples: "5" 22 | no-appendfsync-on-rewrite: "no" 23 | notify-keyspace-events: "" 24 | set-max-intset-entries: "512" 25 | slowlog-log-slower-than: "10000" 26 | slowlog-max-len: "128" 27 | stop-writes-on-bgsave-error: "yes" 28 | tcp-keepalive: "0" 29 | timeout: "0" 30 | zset-max-ziplist-entries: "128" 31 | zset-max-ziplist-value: "64" 32 | -------------------------------------------------------------------------------- /config/samples/example/custom-password.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | annotations: 5 | # if your operator run as cluster-scoped, add this annotations 6 | redis.kun/scope: cluster-scoped 7 | name: mysecret 8 | type: Opaque 9 | data: 10 | password: MWYyZDFlMmU2N2Rm 11 | --- 12 | apiVersion: redis.kun/v1alpha1 13 | kind: DistributedRedisCluster 14 | metadata: 15 | name: example-distributedrediscluster 16 | spec: 17 | image: redis:5.0.4-alpine 18 | masterSize: 3 19 | clusterReplicas: 1 20 | passwordSecret: 21 | name: mysecret 22 | resources: 23 | limits: 24 | cpu: 200m 25 | memory: 200Mi 26 | requests: 27 | cpu: 200m 28 | memory: 100Mi 29 | -------------------------------------------------------------------------------- /config/samples/example/custom-resources.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: DistributedRedisCluster 3 | metadata: 4 | annotations: 5 | # if your operator run as cluster-scoped, add this annotations 6 | redis.kun/scope: cluster-scoped 7 | name: example-distributedrediscluster 8 | spec: 9 | image: redis:5.0.4-alpine 10 | masterSize: 3 11 | clusterReplicas: 1 12 | resources: 13 | limits: 14 | cpu: 200m 15 | memory: 200Mi 16 | requests: 17 | cpu: 200m 18 | memory: 100Mi -------------------------------------------------------------------------------- /config/samples/example/custom-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: DistributedRedisCluster 3 | metadata: 4 | annotations: 5 | # if your operator run as cluster-scoped, add this annotations 6 | redis.kun/scope: cluster-scoped 7 | name: example-distributedrediscluster 8 | spec: 9 | image: redis:5.0.4-alpine 10 | masterSize: 3 11 | clusterReplicas: 1 12 | serviceName: redis-svc -------------------------------------------------------------------------------- /config/samples/example/persistent.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: DistributedRedisCluster 3 | metadata: 4 | annotations: 5 | # if your operator run as cluster-scoped, add this annotations 6 | redis.kun/scope: cluster-scoped 7 | name: example-distributedrediscluster 8 | spec: 9 | image: redis:5.0.4-alpine 10 | masterSize: 3 11 | clusterReplicas: 1 12 | storage: 13 | type: persistent-claim 14 | size: 1Gi 15 | class: csi-rbd-sc 16 | deleteClaim: true -------------------------------------------------------------------------------- /config/samples/example/prometheus-exporter.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: DistributedRedisCluster 3 | metadata: 4 | annotations: 5 | # if your operator run as cluster-scoped, add this annotations 6 | redis.kun/scope: cluster-scoped 7 | name: example-distributedrediscluster 8 | spec: 9 | image: redis:5.0.4-alpine 10 | masterSize: 3 11 | clusterReplicas: 1 12 | monitor: 13 | image: oliver006/redis_exporter -------------------------------------------------------------------------------- /config/samples/example/redis.kun_v1alpha1_distributedrediscluster_cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: DistributedRedisCluster 3 | metadata: 4 | annotations: 5 | # if your operator run as cluster-scoped, add this annotations 6 | redis.kun/scope: cluster-scoped 7 | name: example-distributedrediscluster 8 | spec: 9 | # Add fields here 10 | masterSize: 3 11 | clusterReplicas: 1 12 | image: redis:5.0.4-alpine 13 | -------------------------------------------------------------------------------- /config/samples/example/securitycontext.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: DistributedRedisCluster 3 | metadata: 4 | annotations: 5 | # if your operator run as cluster-scoped, add this annotations 6 | redis.kun/scope: cluster-scoped 7 | name: example-distributedrediscluster 8 | spec: 9 | image: redis:5.0.4-alpine 10 | masterSize: 3 11 | clusterReplicas: 1 12 | securityContext: 13 | runAsUser: 1101 14 | runAsGroup: 1101 15 | fsGroup: 1101 16 | supplementalGroups: [1101] 17 | containerSecurityContext: 18 | allowPrivilegeEscalation: false 19 | capabilities: 20 | drop: 21 | - ALL 22 | -------------------------------------------------------------------------------- /config/samples/kustomization.yaml: -------------------------------------------------------------------------------- 1 | ## Append samples of your project ## 2 | resources: 3 | - redis_v1alpha1_distributedrediscluster.yaml 4 | - redis_v1alpha1_redisclusterbackup.yaml 5 | - redis_v1alpha1_redisclustercleanup.yaml 6 | # +kubebuilder:scaffold:manifestskustomizesamples 7 | -------------------------------------------------------------------------------- /config/samples/redis_v1alpha1_distributedrediscluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: DistributedRedisCluster 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: redis-cluster-operator-old 6 | app.kubernetes.io/managed-by: kustomize 7 | name: distributedrediscluster-sample 8 | spec: 9 | # TODO(user): Add fields here 10 | -------------------------------------------------------------------------------- /config/samples/redis_v1alpha1_redisclusterbackup.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: RedisClusterBackup 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: redis-cluster-operator-old 6 | app.kubernetes.io/managed-by: kustomize 7 | name: redisclusterbackup-sample 8 | spec: 9 | # TODO(user): Add fields here 10 | -------------------------------------------------------------------------------- /config/samples/redis_v1alpha1_redisclustercleanup.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.kun/v1alpha1 2 | kind: RedisClusterCleanup 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: redis-cluster-operator-old 6 | app.kubernetes.io/managed-by: kustomize 7 | name: redisclustercleanup-sample 8 | spec: 9 | schedule: "0 2 * * *" # Cron schedule to run daily at 2 AM. 10 | suspend: false # Set to true to pause the cleanup job. 11 | expiredThreshold: 100 # Minimum number of expired keys to trigger deletion. 12 | scanBatchSize: 100 # Batch size for scanning keys in Redis. 13 | namespaces: 14 | - default # List of namespaces where Redis clusters are deployed. 15 | keyPatterns: 16 | - "apikey-*" 17 | expirationRegexes: 18 | - '"expires":\s*(\d+)' # Regex to extract the expiration timestamp from key values. 19 | skipPatterns: 20 | - "TykJWTSessionID" # Keys containing this string will be skipped. 21 | -------------------------------------------------------------------------------- /config/scorecard/bases/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scorecard.operatorframework.io/v1alpha3 2 | kind: Configuration 3 | metadata: 4 | name: config 5 | stages: 6 | - parallel: true 7 | tests: [] 8 | -------------------------------------------------------------------------------- /config/scorecard/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - bases/config.yaml 3 | apiVersion: kustomize.config.k8s.io/v1beta1 4 | kind: Kustomization 5 | patches: 6 | - path: patches/basic.config.yaml 7 | target: 8 | group: scorecard.operatorframework.io 9 | kind: Configuration 10 | name: config 11 | version: v1alpha3 12 | - path: patches/olm.config.yaml 13 | target: 14 | group: scorecard.operatorframework.io 15 | kind: Configuration 16 | name: config 17 | version: v1alpha3 18 | # +kubebuilder:scaffold:patches 19 | -------------------------------------------------------------------------------- /config/scorecard/patches/basic.config.yaml: -------------------------------------------------------------------------------- 1 | - op: add 2 | path: /stages/0/tests/- 3 | value: 4 | entrypoint: 5 | - scorecard-test 6 | - basic-check-spec 7 | image: quay.io/operator-framework/scorecard-test:v1.39.2 8 | labels: 9 | suite: basic 10 | test: basic-check-spec-test 11 | -------------------------------------------------------------------------------- /config/scorecard/patches/olm.config.yaml: -------------------------------------------------------------------------------- 1 | - op: add 2 | path: /stages/0/tests/- 3 | value: 4 | entrypoint: 5 | - scorecard-test 6 | - olm-bundle-validation 7 | image: quay.io/operator-framework/scorecard-test:v1.39.2 8 | labels: 9 | suite: olm 10 | test: olm-bundle-validation-test 11 | - op: add 12 | path: /stages/0/tests/- 13 | value: 14 | entrypoint: 15 | - scorecard-test 16 | - olm-crds-have-validation 17 | image: quay.io/operator-framework/scorecard-test:v1.39.2 18 | labels: 19 | suite: olm 20 | test: olm-crds-have-validation-test 21 | - op: add 22 | path: /stages/0/tests/- 23 | value: 24 | entrypoint: 25 | - scorecard-test 26 | - olm-crds-have-resources 27 | image: quay.io/operator-framework/scorecard-test:v1.39.2 28 | labels: 29 | suite: olm 30 | test: olm-crds-have-resources-test 31 | - op: add 32 | path: /stages/0/tests/- 33 | value: 34 | entrypoint: 35 | - scorecard-test 36 | - olm-spec-descriptors 37 | image: quay.io/operator-framework/scorecard-test:v1.39.2 38 | labels: 39 | suite: olm 40 | test: olm-spec-descriptors-test 41 | - op: add 42 | path: /stages/0/tests/- 43 | value: 44 | entrypoint: 45 | - scorecard-test 46 | - olm-status-descriptors 47 | image: quay.io/operator-framework/scorecard-test:v1.39.2 48 | labels: 49 | suite: olm 50 | test: olm-status-descriptors-test 51 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2025. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /internal/clustering/placement_v2.go: -------------------------------------------------------------------------------- 1 | package clustering 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/go-logr/logr" 7 | 8 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 9 | "github.com/TykTechnologies/redis-cluster-operator/internal/redisutil" 10 | "github.com/TykTechnologies/redis-cluster-operator/internal/resources/statefulsets" 11 | ) 12 | 13 | type Ctx struct { 14 | log logr.Logger 15 | expectedMasterNum int 16 | clusterName string 17 | cluster *redisutil.Cluster 18 | nodes map[string]redisutil.Nodes 19 | currentMasters redisutil.Nodes 20 | newMastersBySts map[string]*redisutil.Node 21 | slavesByMaster map[string]redisutil.Nodes 22 | bestEffort bool 23 | } 24 | 25 | func NewCtx(cluster *redisutil.Cluster, nodes redisutil.Nodes, masterNum int32, clusterName string, log logr.Logger) *Ctx { 26 | ctx := &Ctx{ 27 | log: log, 28 | expectedMasterNum: int(masterNum), 29 | clusterName: clusterName, 30 | cluster: cluster, 31 | slavesByMaster: make(map[string]redisutil.Nodes), 32 | newMastersBySts: make(map[string]*redisutil.Node), 33 | } 34 | ctx.nodes = ctx.sortRedisNodeByStatefulSet(nodes) 35 | return ctx 36 | } 37 | 38 | func (c *Ctx) sortRedisNodeByStatefulSet(nodes redisutil.Nodes) map[string]redisutil.Nodes { 39 | nodesByStatefulSet := make(map[string]redisutil.Nodes) 40 | 41 | for _, rNode := range nodes { 42 | cNode, err := c.cluster.GetNodeByID(rNode.ID) 43 | if err != nil { 44 | c.log.Error(err, "[sortRedisNodeByStatefulSet] unable fo found the Cluster.Node with redis", "ID", rNode.ID) 45 | continue // if not then next line with cNode.Pod will cause a panic since cNode is nil 46 | } 47 | ssName := unknownVMName 48 | if cNode.StatefulSet != "" { 49 | ssName = cNode.StatefulSet 50 | } 51 | if _, ok := nodesByStatefulSet[ssName]; !ok { 52 | nodesByStatefulSet[ssName] = redisutil.Nodes{} 53 | } 54 | nodesByStatefulSet[ssName] = append(nodesByStatefulSet[ssName], rNode) 55 | if (rNode.GetRole() == redisv1alpha1.RedisClusterNodeRoleMaster) && rNode.TotalSlots() > 0 { 56 | c.currentMasters = append(c.currentMasters, rNode) 57 | } 58 | } 59 | 60 | return nodesByStatefulSet 61 | } 62 | 63 | func (c *Ctx) DispatchMasters() error { 64 | for i := 0; i < c.expectedMasterNum; i++ { 65 | stsName := statefulsets.ClusterStatefulSetName(c.clusterName, i) 66 | nodes, ok := c.nodes[stsName] 67 | if !ok { 68 | return fmt.Errorf("missing statefulset %s", stsName) 69 | } 70 | currentMasterNodes := nodes.FilterByFunc(redisutil.IsMasterWithSlot) 71 | if len(currentMasterNodes) == 0 { 72 | master := c.PlaceMasters(stsName) 73 | c.newMastersBySts[stsName] = master 74 | } else if len(currentMasterNodes) == 1 { 75 | c.newMastersBySts[stsName] = currentMasterNodes[0] 76 | } else if len(currentMasterNodes) > 1 { 77 | c.log.Error(fmt.Errorf("split brain"), "fix manually", "statefulSet", stsName, "masters", currentMasterNodes) 78 | return fmt.Errorf("split brain: %s", stsName) 79 | } 80 | } 81 | 82 | return nil 83 | } 84 | 85 | func (c *Ctx) PlaceMasters(ssName string) *redisutil.Node { 86 | var allMasters redisutil.Nodes 87 | allMasters = append(allMasters, c.currentMasters...) 88 | for _, master := range c.newMastersBySts { 89 | allMasters = append(allMasters, master) 90 | } 91 | nodes := c.nodes[ssName] 92 | for _, cNode := range nodes { 93 | _, err := allMasters.GetNodesByFunc(func(node *redisutil.Node) bool { 94 | return node.NodeName == cNode.NodeName 95 | }) 96 | if err != nil { 97 | return cNode 98 | } 99 | } 100 | c.bestEffort = true 101 | c.log.Info("the pod are not spread enough on VMs to have only one master by VM", "select", nodes[0].IP) 102 | return nodes[0] 103 | } 104 | 105 | func (c *Ctx) PlaceSlaves() error { 106 | c.bestEffort = true 107 | for ssName, nodes := range c.nodes { 108 | master := c.newMastersBySts[ssName] 109 | for _, node := range nodes { 110 | if node.IP == master.IP { 111 | continue 112 | } 113 | if node.NodeName != master.NodeName { 114 | c.bestEffort = false 115 | } 116 | if node.GetRole() == redisv1alpha1.RedisClusterNodeRoleSlave { 117 | if node.MasterReferent != master.ID { 118 | c.log.Error(nil, "master referent conflict", "node ip", node.IP, 119 | "current masterID", node.MasterReferent, "expect masterID", master.ID, "master IP", master.IP) 120 | c.slavesByMaster[master.ID] = append(c.slavesByMaster[master.ID], node) 121 | } 122 | continue 123 | } 124 | c.slavesByMaster[master.ID] = append(c.slavesByMaster[master.ID], node) 125 | } 126 | } 127 | return nil 128 | } 129 | 130 | func (c *Ctx) GetCurrentMasters() redisutil.Nodes { 131 | return c.currentMasters 132 | } 133 | 134 | func (c *Ctx) GetNewMasters() redisutil.Nodes { 135 | var nodes redisutil.Nodes 136 | for _, node := range c.newMastersBySts { 137 | nodes = append(nodes, node) 138 | } 139 | return nodes 140 | } 141 | 142 | func (c *Ctx) GetSlaves() map[string]redisutil.Nodes { 143 | return c.slavesByMaster 144 | } 145 | 146 | func (c *Ctx) GetStatefulsetNodes() map[string]redisutil.Nodes { 147 | return c.nodes 148 | } 149 | -------------------------------------------------------------------------------- /internal/clustering/rebalance.go: -------------------------------------------------------------------------------- 1 | package clustering 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | 7 | "github.com/TykTechnologies/redis-cluster-operator/internal/redisutil" 8 | "github.com/TykTechnologies/redis-cluster-operator/internal/utils" 9 | ) 10 | 11 | // RebalancedCluster rebalanced a redis cluster. 12 | func (c *Ctx) RebalancedCluster(admin redisutil.IAdmin, newMasterNodes redisutil.Nodes) error { 13 | nbNode := len(newMasterNodes) 14 | for _, node := range newMasterNodes { 15 | expected := int(float64(admin.GetHashMaxSlot()+1) / float64(nbNode)) 16 | node.SetBalance(len(node.Slots) - expected) 17 | } 18 | 19 | totalBalance := 0 20 | for _, node := range newMasterNodes { 21 | totalBalance += node.Balance() 22 | } 23 | 24 | for totalBalance > 0 { 25 | for _, node := range newMasterNodes { 26 | if node.Balance() < 0 && totalBalance > 0 { 27 | b := node.Balance() - 1 28 | node.SetBalance(b) 29 | totalBalance -= 1 30 | } 31 | } 32 | } 33 | 34 | // Sort nodes by their slots balance. 35 | sn := newMasterNodes.SortByFunc(func(a, b *redisutil.Node) bool { return a.Balance() < b.Balance() }) 36 | if log.V(4).Enabled() { 37 | for _, node := range sn { 38 | log.Info("debug rebalanced master", "node", node.IPPort(), "balance", node.Balance()) 39 | } 40 | } 41 | 42 | log.Info(">>> rebalancing", "nodeNum", nbNode) 43 | 44 | dstIdx := 0 45 | srcIdx := len(sn) - 1 46 | 47 | for dstIdx < srcIdx { 48 | dst := sn[dstIdx] 49 | src := sn[srcIdx] 50 | 51 | var numSlots float64 52 | if math.Abs(float64(dst.Balance())) < math.Abs(float64(src.Balance())) { 53 | numSlots = math.Abs(float64(dst.Balance())) 54 | } else { 55 | numSlots = math.Abs(float64(src.Balance())) 56 | } 57 | 58 | if numSlots > 0 { 59 | log.Info(fmt.Sprintf("Moving %f slots from %s to %s", numSlots, src.IPPort(), dst.IPPort())) 60 | srcs := redisutil.Nodes{src} 61 | reshardTable := computeReshardTable(srcs, int(numSlots)) 62 | if len(reshardTable) != int(numSlots) { 63 | log.Error(nil, "*** Assertion failed: Reshard table != number of slots", "table", len(reshardTable), "slots", numSlots) 64 | } 65 | for _, e := range reshardTable { 66 | if err := c.moveSlot(e, dst, admin); err != nil { 67 | return err 68 | } 69 | } 70 | } 71 | 72 | // Update nodes balance. 73 | log.V(4).Info("balance", "dst", dst.Balance(), "src", src.Balance(), "slots", numSlots) 74 | dst.SetBalance(dst.Balance() + int(numSlots)) 75 | src.SetBalance(src.Balance() - int(numSlots)) 76 | if dst.Balance() == 0 { 77 | dstIdx += 1 78 | } 79 | if src.Balance() == 0 { 80 | srcIdx -= 1 81 | } 82 | } 83 | 84 | return nil 85 | } 86 | 87 | type MovedNode struct { 88 | Source *redisutil.Node 89 | Slot redisutil.Slot 90 | } 91 | 92 | // computeReshardTable Given a list of source nodes return a "resharding plan" 93 | // with what slots to move in order to move "numslots" slots to another instance. 94 | func computeReshardTable(src redisutil.Nodes, numSlots int) []*MovedNode { 95 | var moved []*MovedNode 96 | 97 | sources := src.SortByFunc(func(a, b *redisutil.Node) bool { return a.TotalSlots() < b.TotalSlots() }) 98 | sourceTotSlots := 0 99 | for _, node := range sources { 100 | sourceTotSlots += node.TotalSlots() 101 | } 102 | for idx, node := range sources { 103 | n := float64(numSlots) / float64(sourceTotSlots) * float64(node.TotalSlots()) 104 | 105 | if idx == 0 { 106 | n = math.Ceil(n) 107 | } else { 108 | n = math.Floor(n) 109 | } 110 | 111 | keys := node.Slots 112 | 113 | for i := 0; i < int(n); i++ { 114 | if len(moved) < numSlots { 115 | mnode := &MovedNode{ 116 | Source: node, 117 | Slot: keys[i], 118 | } 119 | moved = append(moved, mnode) 120 | } 121 | } 122 | } 123 | return moved 124 | } 125 | 126 | func (c *Ctx) moveSlot(source *MovedNode, target *redisutil.Node, admin redisutil.IAdmin) error { 127 | if err := admin.SetSlot(target.IPPort(), "IMPORTING", source.Slot, target.ID); err != nil { 128 | return err 129 | } 130 | if err := admin.SetSlot(source.Source.IPPort(), "MIGRATING", source.Slot, source.Source.ID); err != nil { 131 | return err 132 | } 133 | if _, err := admin.MigrateKeysInSlot(source.Source.IPPort(), target, source.Slot, 10, 30000, true); err != nil { 134 | return err 135 | } 136 | if err := admin.SetSlot(target.IPPort(), "NODE", source.Slot, target.ID); err != nil { 137 | c.log.Error(err, "SET NODE", "node", target.IPPort()) 138 | } 139 | if err := admin.SetSlot(source.Source.IPPort(), "NODE", source.Slot, target.ID); err != nil { 140 | c.log.Error(err, "SET NODE", "node", source.Source.IPPort()) 141 | } 142 | source.Source.Slots = redisutil.RemoveSlot(source.Source.Slots, source.Slot) 143 | return nil 144 | } 145 | 146 | func (c *Ctx) AllocSlots(admin redisutil.IAdmin, newMasterNodes redisutil.Nodes) error { 147 | mastersNum := len(newMasterNodes) 148 | clusterHashSlots := int(admin.GetHashMaxSlot() + 1) 149 | slotsPerNode := float64(clusterHashSlots) / float64(mastersNum) 150 | first := 0 151 | cursor := 0.0 152 | for index, node := range newMasterNodes { 153 | last := utils.Round(cursor + slotsPerNode - 1) 154 | if last > clusterHashSlots || index == mastersNum-1 { 155 | last = clusterHashSlots - 1 156 | } 157 | 158 | if last < first { 159 | last = first 160 | } 161 | 162 | node.Slots = redisutil.BuildSlotSlice(redisutil.Slot(first), redisutil.Slot(last)) 163 | first = last + 1 164 | cursor += slotsPerNode 165 | if err := admin.AddSlots(node.IPPort(), node.Slots); err != nil { 166 | return err 167 | } 168 | } 169 | return nil 170 | } 171 | -------------------------------------------------------------------------------- /internal/clustering/rebalance_test.go: -------------------------------------------------------------------------------- 1 | package clustering 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/TykTechnologies/redis-cluster-operator/internal/redisutil" 7 | ) 8 | 9 | func Test_computeReshardTable(t *testing.T) { 10 | type args struct { 11 | src redisutil.Nodes 12 | numSlots int 13 | } 14 | tests := []struct { 15 | name string 16 | args args 17 | want int 18 | }{ 19 | { 20 | name: "", 21 | args: args{ 22 | src: redisutil.Nodes{&redisutil.Node{ 23 | ID: "node1", 24 | IP: "10.1.1.1", 25 | Port: "6379", 26 | Role: "master", 27 | Slots: redisutil.BuildSlotSlice(5461, 10922), 28 | }}, 29 | numSlots: 1366, 30 | }, 31 | want: 1366, 32 | }, 33 | } 34 | for _, tt := range tests { 35 | t.Run(tt.name, func(t *testing.T) { 36 | if got := computeReshardTable(tt.args.src, tt.args.numSlots); len(got) != tt.want { 37 | t.Errorf("computeReshardTable() = %v, want %v", len(got), tt.want) 38 | } 39 | }) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /internal/clustering/roles.go: -------------------------------------------------------------------------------- 1 | package clustering 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/TykTechnologies/redis-cluster-operator/internal/redisutil" 7 | ) 8 | 9 | // AttachingSlavesToMaster used to attach slaves to their masters 10 | func (c *Ctx) AttachingSlavesToMaster(admin redisutil.IAdmin) error { 11 | var globalErr error 12 | for masterID, slaves := range c.slavesByMaster { 13 | masterNode, err := c.cluster.GetNodeByID(masterID) 14 | if err != nil { 15 | c.log.Error(err, fmt.Sprintf("unable fo found the Cluster.Node with redis ID:%s", masterID)) 16 | continue 17 | } 18 | for _, slave := range slaves { 19 | c.log.Info(fmt.Sprintf("attaching node %s to master %s", slave.ID, masterID)) 20 | 21 | err := admin.AttachSlaveToMaster(slave, masterNode.ID) 22 | if err != nil { 23 | c.log.Error(err, fmt.Sprintf("attaching node %s to master %s", slave.ID, masterID)) 24 | globalErr = err 25 | } 26 | } 27 | } 28 | return globalErr 29 | } 30 | -------------------------------------------------------------------------------- /internal/config/redis.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "path" 5 | 6 | "github.com/spf13/pflag" 7 | ) 8 | 9 | const ( 10 | // DefaultRedisTimeout default redis timeout (ms) 11 | DefaultRedisTimeout = 2000 12 | //DefaultClusterNodeTimeout default cluster node timeout (ms) 13 | //The maximum amount of time a Redis Cluster node can be unavailable, without it being considered as failing 14 | DefaultClusterNodeTimeout = 2000 15 | // RedisRenameCommandsDefaultPath default path to volume storing rename commands 16 | RedisRenameCommandsDefaultPath = "/etc/secret-volume" 17 | // RedisRenameCommandsDefaultFile default file name containing rename commands 18 | RedisRenameCommandsDefaultFile = "" 19 | // RedisConfigFileDefault default config file path 20 | RedisConfigFileDefault = "/redis-conf/redis.conf" 21 | // RedisServerBinDefault default binary name 22 | RedisServerBinDefault = "redis-server" 23 | // RedisServerPortDefault default redis port 24 | RedisServerPortDefault = "6379" 25 | // RedisMaxMemoryDefault default redis max memory 26 | RedisMaxMemoryDefault = 0 27 | // RedisMaxMemoryPolicyDefault default redis max memory evition policy 28 | RedisMaxMemoryPolicyDefault = "noeviction" 29 | ) 30 | 31 | //var redisFlagSet *pflag.FlagSet 32 | // 33 | //func init() { 34 | // redisFlagSet = pflag.NewFlagSet("redis", pflag.ExitOnError) 35 | //} 36 | 37 | var redisConf *Redis 38 | 39 | func init() { 40 | redisConf = &Redis{} 41 | } 42 | 43 | func RedisConf() *Redis { 44 | return redisConf 45 | } 46 | 47 | // Redis used to store all Redis configuration information 48 | type Redis struct { 49 | DialTimeout int 50 | ClusterNodeTimeout int 51 | ConfigFileName string 52 | RenameCommandsPath string 53 | RenameCommandsFile string 54 | HTTPServerAddr string 55 | ServerBin string 56 | ServerPort string 57 | ServerIP string 58 | MaxMemory uint32 59 | MaxMemoryPolicy string 60 | ConfigFiles []string 61 | } 62 | 63 | // AddFlags use to add the Redis Config flags to the command line 64 | func (r *Redis) AddFlags(fs *pflag.FlagSet) { 65 | fs.IntVar(&r.DialTimeout, "rdt", DefaultRedisTimeout, "redis dial timeout (ms)") 66 | fs.IntVar(&r.ClusterNodeTimeout, "cluster-node-timeout", DefaultClusterNodeTimeout, "redis node timeout (ms)") 67 | fs.StringVar(&r.ConfigFileName, "c", RedisConfigFileDefault, "redis config file path") 68 | fs.StringVar(&r.RenameCommandsPath, "rename-command-path", RedisRenameCommandsDefaultPath, "Path to the folder where rename-commands option for redis are available") 69 | fs.StringVar(&r.RenameCommandsFile, "rename-command-file", RedisRenameCommandsDefaultFile, "Name of the file where rename-commands option for redis are available, disabled if empty") 70 | fs.Uint32Var(&r.MaxMemory, "max-memory", RedisMaxMemoryDefault, "redis max memory") 71 | fs.StringVar(&r.MaxMemoryPolicy, "max-memory-policy", RedisMaxMemoryPolicyDefault, "redis max memory evition policy") 72 | fs.StringVar(&r.ServerBin, "bin", RedisServerBinDefault, "redis server binary file name") 73 | fs.StringVar(&r.ServerPort, "port", RedisServerPortDefault, "redis server listen port") 74 | fs.StringVar(&r.ServerIP, "ip", "", "redis server listen ip") 75 | fs.StringArrayVar(&r.ConfigFiles, "config-file", []string{}, "Location of redis configuration file that will be include in the ") 76 | } 77 | 78 | // GetRenameCommandsFile return the path to the rename command file, or empty string if not define 79 | func (r *Redis) GetRenameCommandsFile() string { 80 | if r.RenameCommandsFile == "" { 81 | return "" 82 | } 83 | return path.Join(r.RenameCommandsPath, r.RenameCommandsFile) 84 | } 85 | -------------------------------------------------------------------------------- /internal/controller/errors.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | // ErrorType is the type of an error 10 | type ErrorType uint 11 | 12 | const ( 13 | // NoType error 14 | NoType ErrorType = iota 15 | // Requeue error 16 | Requeue 17 | // Kubernetes error 18 | Kubernetes 19 | // Redis error 20 | Redis 21 | // Cluster 22 | Cluster 23 | // StopRetry stop retry error 24 | StopRetry 25 | ) 26 | 27 | type customError struct { 28 | errorType ErrorType 29 | originalError error 30 | } 31 | 32 | // New creates a new customError 33 | func (errorType ErrorType) New(msg string) error { 34 | return customError{errorType: errorType, originalError: errors.New(msg)} 35 | } 36 | 37 | // New creates a new customError with formatted message 38 | func (errorType ErrorType) Newf(msg string, args ...interface{}) error { 39 | return customError{errorType: errorType, originalError: fmt.Errorf(msg, args...)} 40 | } 41 | 42 | // Wrap creates a new wrapped error 43 | func (errorType ErrorType) Wrap(err error, msg string) error { 44 | return errorType.Wrapf(err, msg) 45 | } 46 | 47 | // Wrap creates a new wrapped error with formatted message 48 | func (errorType ErrorType) Wrapf(err error, msg string, args ...interface{}) error { 49 | return customError{errorType: errorType, originalError: errors.Wrapf(err, msg, args...)} 50 | } 51 | 52 | // Error returns the mssage of a customError 53 | func (error customError) Error() string { 54 | return error.originalError.Error() 55 | } 56 | 57 | // New creates a no type error 58 | func New(msg string) error { 59 | return customError{errorType: NoType, originalError: errors.New(msg)} 60 | } 61 | 62 | // Newf creates a no type error with formatted message 63 | func Newf(msg string, args ...interface{}) error { 64 | return customError{errorType: NoType, originalError: errors.New(fmt.Sprintf(msg, args...))} 65 | } 66 | 67 | // Wrap an error with a string 68 | func Wrap(err error, msg string) error { 69 | return Wrapf(err, msg) 70 | } 71 | 72 | // Cause gives the original error 73 | func Cause(err error) error { 74 | return errors.Cause(err) 75 | } 76 | 77 | // Wrapf an error with format string 78 | func Wrapf(err error, msg string, args ...interface{}) error { 79 | wrappedError := errors.Wrapf(err, msg, args...) 80 | if customErr, ok := err.(customError); ok { 81 | return customError{ 82 | errorType: customErr.errorType, 83 | originalError: wrappedError, 84 | } 85 | } 86 | 87 | return customError{errorType: NoType, originalError: wrappedError} 88 | } 89 | 90 | // GetType returns the error type 91 | func GetType(err error) ErrorType { 92 | if customErr, ok := err.(customError); ok { 93 | return customErr.errorType 94 | } 95 | 96 | return NoType 97 | } 98 | -------------------------------------------------------------------------------- /internal/controller/redisclusterbackup/helper.go: -------------------------------------------------------------------------------- 1 | package redisclusterbackup 2 | 3 | import ( 4 | "context" 5 | 6 | batch "k8s.io/api/batch/v1" 7 | corev1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/client-go/rest" 10 | "sigs.k8s.io/controller-runtime/pkg/client" 11 | 12 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 13 | ) 14 | 15 | func (r *RedisClusterBackupReconciler) markAsFailedBackup(backup *redisv1alpha1.RedisClusterBackup, 16 | reason string) error { 17 | t := metav1.Now() 18 | backup.Status.CompletionTime = &t 19 | backup.Status.Phase = redisv1alpha1.BackupPhaseFailed 20 | backup.Status.Reason = reason 21 | return r.crController.UpdateCRStatus(backup) 22 | } 23 | 24 | func (r *RedisClusterBackupReconciler) markAsIgnoredBackup(backup *redisv1alpha1.RedisClusterBackup, 25 | reason string) error { 26 | t := metav1.Now() 27 | backup.Status.CompletionTime = &t 28 | backup.Status.Phase = redisv1alpha1.BackupPhaseIgnored 29 | backup.Status.Reason = reason 30 | return r.crController.UpdateCRStatus(backup) 31 | } 32 | 33 | func (r *RedisClusterBackupReconciler) isBackupRunning(backup *redisv1alpha1.RedisClusterBackup) (bool, error) { 34 | labMap := client.MatchingLabels{ 35 | redisv1alpha1.LabelBackupStatus: string(redisv1alpha1.BackupPhaseRunning), 36 | redisv1alpha1.LabelClusterName: backup.Spec.RedisClusterName, 37 | } 38 | backupList := &redisv1alpha1.RedisClusterBackupList{} 39 | opts := []client.ListOption{ 40 | client.InNamespace(backup.Namespace), 41 | labMap, 42 | } 43 | err := r.Client.List(context.TODO(), backupList, opts...) 44 | if err != nil { 45 | return false, err 46 | } 47 | 48 | jobLabMap := client.MatchingLabels{ 49 | redisv1alpha1.LabelClusterName: backup.Spec.RedisClusterName, 50 | redisv1alpha1.AnnotationJobType: redisv1alpha1.JobTypeBackup, 51 | } 52 | backupJobList, err := r.jobController.ListJobByLabels(backup.Namespace, jobLabMap) 53 | if err != nil { 54 | return false, err 55 | } 56 | 57 | if len(backupList.Items) > 0 && len(backupJobList.Items) > 0 { 58 | return true, nil 59 | } 60 | 61 | return false, nil 62 | } 63 | 64 | func upsertEnvVars(vars []corev1.EnvVar, nv ...corev1.EnvVar) []corev1.EnvVar { 65 | upsert := func(env corev1.EnvVar) { 66 | for i, v := range vars { 67 | if v.Name == env.Name { 68 | vars[i] = env 69 | return 70 | } 71 | } 72 | vars = append(vars, env) 73 | } 74 | 75 | for _, env := range nv { 76 | upsert(env) 77 | } 78 | return vars 79 | } 80 | 81 | // Returns the REDIS_PASSWORD environment variable. 82 | func redisPassword(cluster *redisv1alpha1.DistributedRedisCluster) corev1.EnvVar { 83 | secretName := cluster.Spec.PasswordSecret.Name 84 | return corev1.EnvVar{ 85 | Name: "REDIS_PASSWORD", 86 | ValueFrom: &corev1.EnvVarSource{ 87 | SecretKeyRef: &corev1.SecretKeySelector{ 88 | LocalObjectReference: corev1.LocalObjectReference{ 89 | Name: secretName, 90 | }, 91 | Key: "password", 92 | }, 93 | }, 94 | } 95 | } 96 | 97 | func newDirectClient(config *rest.Config) client.Client { 98 | c, err := client.New(config, client.Options{}) 99 | if err != nil { 100 | panic(err) 101 | } 102 | return c 103 | } 104 | 105 | func isJobFinished(j *batch.Job) bool { 106 | for _, c := range j.Status.Conditions { 107 | if (c.Type == batch.JobComplete || c.Type == batch.JobFailed) && c.Status == corev1.ConditionTrue { 108 | return true 109 | } 110 | } 111 | return false 112 | } 113 | -------------------------------------------------------------------------------- /internal/controller/redisclustercleanup/cleaner.go: -------------------------------------------------------------------------------- 1 | package redisclustercleanup 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "regexp" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "github.com/go-logr/logr" 12 | "github.com/go-redis/redis/v8" 13 | 14 | "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 15 | ) 16 | 17 | // processHost connects to the given Redis host and cleans up expired keys. 18 | func processHost(host, port, password string, spec v1alpha1.RedisClusterCleanupSpec, logger logr.Logger) { 19 | logger.Info("processing", "host", host, "port", port) 20 | ctx := context.Background() 21 | addr := host + ":" + port 22 | 23 | // Create a Redis client for this host. 24 | client := redis.NewClient(&redis.Options{ 25 | Addr: addr, 26 | Password: password, 27 | DB: 0, 28 | }) 29 | defer func() { 30 | if err := client.Close(); err != nil { 31 | logger.Error(err, "Error closing Redis client", "node", addr) 32 | } 33 | }() 34 | 35 | cleanupTriggered := false 36 | // Pre-compile expiration regexes provided in the spec. 37 | var expirationRegexList []*regexp.Regexp 38 | for _, regexStr := range spec.ExpirationRegexes { 39 | re, err := regexp.Compile(regexStr) 40 | if err != nil { 41 | logger.Error(err, "Invalid expiration regex", "regex", regexStr) 42 | continue 43 | } 44 | expirationRegexList = append(expirationRegexList, re) 45 | } 46 | 47 | scanBatchSize := int64(spec.ScanBatchSize) 48 | expiredThreshold := int64(spec.ExpiredThreshold) 49 | var keysToDelete []string 50 | 51 | // Iterate over each key pattern specified in the spec. 52 | for _, keyPattern := range spec.KeyPatterns { 53 | var cursor uint64 = 0 54 | for { 55 | keys, newCursor, err := client.Scan(ctx, cursor, keyPattern, scanBatchSize).Result() 56 | if err != nil { 57 | logger.Error(err, "Error scanning keys", "node", addr, "pattern", keyPattern) 58 | break 59 | } 60 | cursor = newCursor 61 | 62 | if len(keys) == 0 { 63 | if cursor == 0 { 64 | break 65 | } 66 | continue 67 | } 68 | 69 | // Get the key values via a pipeline. 70 | pipe := client.Pipeline() 71 | cmds := make([]*redis.StringCmd, len(keys)) 72 | for i, key := range keys { 73 | cmds[i] = pipe.Get(ctx, key) 74 | } 75 | _, err = pipe.Exec(ctx) 76 | if err != nil && !errors.Is(err, redis.Nil) { 77 | logger.Error(err, "Error executing pipeline GET", "node", addr) 78 | } 79 | 80 | now := time.Now().Unix() 81 | // Process each key's value. 82 | for i, key := range keys { 83 | value, err := cmds[i].Result() 84 | if err != nil { 85 | // Skip keys that are not found; log other errors. 86 | if !errors.Is(err, redis.Nil) { 87 | logger.Info("Error getting value for key", "key", key, "node", addr) 88 | } 89 | continue 90 | } 91 | 92 | // Skip key if it contains any of the skip patterns. 93 | skipKey := false 94 | for _, skipPattern := range spec.SkipPatterns { 95 | if strings.Contains(value, skipPattern) { 96 | skipKey = true 97 | break 98 | } 99 | } 100 | if skipKey { 101 | continue 102 | } 103 | 104 | // Use the compiled expiration regexes to extract an expiration. 105 | var keyExpires int64 106 | matched := false 107 | for _, re := range expirationRegexList { 108 | matches := re.FindStringSubmatch(value) 109 | if len(matches) < 2 { 110 | continue 111 | } 112 | keyExpires, err = strconv.ParseInt(matches[1], 10, 64) 113 | if err != nil { 114 | logger.Info("Error parsing expires for key", "key", key, "node", addr) 115 | continue 116 | } 117 | matched = true 118 | break // Use the first successful match. 119 | } 120 | if !matched { 121 | // No valid expiration found. 122 | continue 123 | } 124 | 125 | // Check if the key is expired. 126 | if keyExpires > 0 && now > keyExpires { 127 | keysToDelete = append(keysToDelete, key) 128 | } 129 | } 130 | 131 | // Trigger deletion if the number of accumulated expired keys meets/exceeds the threshold. 132 | if int64(len(keysToDelete)) >= expiredThreshold { 133 | logger.Info("Expired keys threshold reached, deleting keys", "redis-host", addr, "pattern", keyPattern, "count", len(keysToDelete)) 134 | cleanupTriggered = true 135 | pipeDel := client.Pipeline() 136 | for _, key := range keysToDelete { 137 | pipeDel.Del(ctx, key) 138 | } 139 | _, err := pipeDel.Exec(ctx) 140 | if err != nil { 141 | logger.Error(err, "Error executing pipeline DEL", "node", addr) 142 | } 143 | // Reset the key accumulator. 144 | keysToDelete = nil 145 | } 146 | 147 | // If SCAN iteration is complete for this pattern. 148 | if cursor == 0 { 149 | break 150 | } 151 | } 152 | } 153 | 154 | // Delete any remaining keys after scanning is complete. 155 | // There can be a situation where the final set of keys accumulated (from the last SCAN iteration) 156 | // does not meet the threshold for batch deletion. If we don't handle these, 157 | // they'd remain undeleted even though they meet the conditions for deletion. 158 | if len(keysToDelete) > 0 && cleanupTriggered { 159 | logger.Info("Final Cleanup: Deleting remaining keys", "redis-host", addr, "count", len(keysToDelete)) 160 | pipeDel := client.Pipeline() 161 | for _, key := range keysToDelete { 162 | pipeDel.Del(ctx, key) 163 | } 164 | _, err := pipeDel.Exec(ctx) 165 | if err != nil { 166 | logger.Error(err, "Error deleting remaining keys", "node", addr) 167 | } 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /internal/event/event.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | const ( 4 | BackupError string = "BakcupError" 5 | BackupFailed string = "BakcupFailed" 6 | Starting string = "Starting" 7 | Successful string = "Successful" 8 | BackupSuccessful string = "SuccessfulBackup" 9 | 10 | CleanupError string = "CleanupError" 11 | ) 12 | -------------------------------------------------------------------------------- /internal/exec/exec.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "net/url" 7 | "strings" 8 | 9 | "github.com/go-logr/logr" 10 | corev1 "k8s.io/api/core/v1" 11 | "k8s.io/client-go/kubernetes/scheme" 12 | "k8s.io/client-go/rest" 13 | "k8s.io/client-go/tools/remotecommand" 14 | ) 15 | 16 | // IExec is an injectable interface for running remote exec commands. 17 | type IExec interface { 18 | // ExecCommandInPodSet exec cmd in pod set. 19 | ExecCommandInPodSet(podSet []*corev1.Pod, cmd ...string) error 20 | } 21 | 22 | type remoteExec struct { 23 | restGVKClient rest.Interface 24 | logger logr.Logger 25 | config *rest.Config 26 | } 27 | 28 | // NewRemoteExec returns a new IExec which will exec remote cmd. 29 | func NewRemoteExec(restGVKClient rest.Interface, config *rest.Config, logger logr.Logger) IExec { 30 | return &remoteExec{ 31 | restGVKClient: restGVKClient, 32 | logger: logger, 33 | config: config, 34 | } 35 | } 36 | 37 | // ExecOptions passed to ExecWithOptions. 38 | type ExecOptions struct { 39 | Command []string 40 | 41 | Namespace string 42 | PodName string 43 | ContainerName string 44 | 45 | Stdin io.Reader 46 | CaptureStdout bool 47 | CaptureStderr bool 48 | // If false, whitespace in std{err,out} will be removed. 49 | PreserveWhitespace bool 50 | } 51 | 52 | // ExecCommandInPodSet implements IExec interface. 53 | func (e *remoteExec) ExecCommandInPodSet(podSet []*corev1.Pod, cmd ...string) error { 54 | for _, pod := range podSet { 55 | if _, err := e.ExecCommandInContainer(pod.Namespace, pod.Name, pod.Spec.Containers[0].Name, cmd...); err != nil { 56 | return err 57 | } 58 | } 59 | return nil 60 | } 61 | 62 | // ExecCommandInContainer executes a command in the specified container. 63 | func (e *remoteExec) ExecCommandInContainer(namespace, podName, containerName string, cmd ...string) (string, error) { 64 | stdout, stderr, err := e.ExecCommandInContainerWithFullOutput(namespace, podName, containerName, cmd...) 65 | if stderr != "" { 66 | e.logger.Info("ExecCommand", "command", cmd, "stderr", stderr) 67 | } 68 | return stdout, err 69 | } 70 | 71 | // ExecCommandInContainerWithFullOutput executes a command in the 72 | // specified container and return stdout, stderr and error 73 | func (e *remoteExec) ExecCommandInContainerWithFullOutput(namespace, podName, containerName string, cmd ...string) (string, string, error) { 74 | return e.ExecWithOptions(ExecOptions{ 75 | Command: cmd, 76 | Namespace: namespace, 77 | PodName: podName, 78 | ContainerName: containerName, 79 | 80 | Stdin: nil, 81 | CaptureStdout: true, 82 | CaptureStderr: true, 83 | PreserveWhitespace: false, 84 | }) 85 | } 86 | 87 | // ExecWithOptions executes a command in the specified container, 88 | // returning stdout, stderr and error. `options` allowed for 89 | // additional parameters to be passed. 90 | func (e *remoteExec) ExecWithOptions(options ExecOptions) (string, string, error) { 91 | const tty = false 92 | 93 | req := e.restGVKClient.Post(). 94 | Resource("pods"). 95 | Name(options.PodName). 96 | Namespace(options.Namespace). 97 | SubResource("exec"). 98 | Param("container", options.ContainerName) 99 | 100 | req.VersionedParams(&corev1.PodExecOptions{ 101 | Container: options.ContainerName, 102 | Command: options.Command, 103 | Stdin: options.Stdin != nil, 104 | Stdout: options.CaptureStdout, 105 | Stderr: options.CaptureStderr, 106 | TTY: tty, 107 | }, scheme.ParameterCodec) 108 | 109 | var stdout, stderr bytes.Buffer 110 | err := execute("POST", req.URL(), e.config, options.Stdin, &stdout, &stderr, tty) 111 | 112 | if options.PreserveWhitespace { 113 | return stdout.String(), stderr.String(), err 114 | } 115 | return strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err 116 | } 117 | 118 | func execute(method string, url *url.URL, config *rest.Config, stdin io.Reader, stdout, stderr io.Writer, tty bool) error { 119 | exec, err := remotecommand.NewSPDYExecutor(config, method, url) 120 | if err != nil { 121 | return err 122 | } 123 | return exec.Stream(remotecommand.StreamOptions{ 124 | Stdin: stdin, 125 | Stdout: stdout, 126 | Stderr: stderr, 127 | Tty: tty, 128 | }) 129 | } 130 | -------------------------------------------------------------------------------- /internal/heal/clustersplit.go: -------------------------------------------------------------------------------- 1 | package heal 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "k8s.io/apimachinery/pkg/util/errors" 8 | 9 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 10 | "github.com/TykTechnologies/redis-cluster-operator/internal/config" 11 | "github.com/TykTechnologies/redis-cluster-operator/internal/redisutil" 12 | ) 13 | 14 | // FixClusterSplit use to detect and fix Cluster split 15 | func (c *CheckAndHeal) FixClusterSplit(cluster *redisv1alpha1.DistributedRedisCluster, infos *redisutil.ClusterInfos, admin redisutil.IAdmin, config *config.Redis) (bool, error) { 16 | clusters := buildClustersLists(infos) 17 | 18 | if len(clusters) > 1 { 19 | if c.DryRun { 20 | return true, nil 21 | } 22 | return true, c.reassignClusters(admin, config, clusters) 23 | } 24 | c.Logger.V(3).Info("[Check] No split cluster detected") 25 | return false, nil 26 | } 27 | 28 | type cluster []string 29 | 30 | func (c *CheckAndHeal) reassignClusters(admin redisutil.IAdmin, config *config.Redis, clusters []cluster) error { 31 | c.Logger.Info("[Check] Cluster split detected, the Redis manager will recover from the issue, but data may be lost") 32 | var errs []error 33 | // only one cluster may remain 34 | mainCluster, badClusters := splitMainCluster(clusters) 35 | if len(mainCluster) == 0 { 36 | c.Logger.Error(nil, "[Check] Impossible to fix cluster split, cannot elect main cluster") 37 | return fmt.Errorf("impossible to fix cluster split, cannot elect main cluster") 38 | } 39 | c.Logger.Info("[Check] Cluster is elected as main cluster", "Cluster", mainCluster) 40 | // reset admin to connect to the correct cluster 41 | admin.Connections().ReplaceAll(mainCluster) 42 | 43 | // reconfigure bad clusters 44 | for _, cluster := range badClusters { 45 | c.Logger.Info(fmt.Sprintf("[Check] All keys stored in redis cluster '%s' will be lost", cluster)) 46 | clusterAdmin := redisutil.NewAdmin(cluster, 47 | &redisutil.AdminOptions{ 48 | ConnectionTimeout: time.Duration(config.DialTimeout) * time.Millisecond, 49 | RenameCommandsFile: config.GetRenameCommandsFile(), 50 | }, c.Logger) 51 | for _, nodeAddr := range cluster { 52 | if err := clusterAdmin.FlushAndReset(nodeAddr, redisutil.ResetHard); err != nil { 53 | c.Logger.Error(err, "unable to flush the node", "node", nodeAddr) 54 | errs = append(errs, err) 55 | } 56 | if err := admin.AttachNodeToCluster(nodeAddr); err != nil { 57 | c.Logger.Error(err, "unable to attach the node", "node", nodeAddr) 58 | errs = append(errs, err) 59 | } 60 | 61 | } 62 | clusterAdmin.Close() 63 | } 64 | 65 | return errors.NewAggregate(errs) 66 | } 67 | 68 | func splitMainCluster(clusters []cluster) (cluster, []cluster) { 69 | if len(clusters) == 0 { 70 | return cluster{}, []cluster{} 71 | } 72 | // only the bigger cluster is kept, or the first one if several clusters have the same size 73 | maincluster := -1 74 | maxSize := 0 75 | for i, c := range clusters { 76 | if len(c) > maxSize { 77 | maxSize = len(c) 78 | maincluster = i 79 | } 80 | } 81 | if maincluster != -1 { 82 | main := clusters[maincluster] 83 | return main, append(clusters[:maincluster], clusters[maincluster+1:]...) 84 | } 85 | return clusters[0], []cluster{} 86 | } 87 | 88 | // buildClustersLists build a list of independent clusters 89 | // we could have cluster partially overlapping in case of inconsistent cluster view 90 | func buildClustersLists(infos *redisutil.ClusterInfos) []cluster { 91 | var clusters []cluster 92 | for _, nodeinfos := range infos.Infos { 93 | if nodeinfos == nil || nodeinfos.Node == nil { 94 | continue 95 | } 96 | slice := append(nodeinfos.Friends, nodeinfos.Node) 97 | var c cluster 98 | // build list of addresses 99 | for _, node := range slice { 100 | if len(node.FailStatus) == 0 { 101 | c = append(c, node.IPPort()) 102 | } 103 | } 104 | // check if this cluster overlap with another 105 | overlap := false 106 | for _, node := range c { 107 | if findInCluster(node, clusters) { 108 | overlap = true 109 | break 110 | } 111 | } 112 | // if this is a new cluster, add it 113 | if !overlap { 114 | clusters = append(clusters, c) 115 | } 116 | } 117 | return clusters 118 | } 119 | 120 | func findInCluster(addr string, clusters []cluster) bool { 121 | for _, c := range clusters { 122 | for _, nodeAddr := range c { 123 | if addr == nodeAddr { 124 | return true 125 | } 126 | } 127 | } 128 | return false 129 | } 130 | -------------------------------------------------------------------------------- /internal/heal/clustersplit_test.go: -------------------------------------------------------------------------------- 1 | package heal 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/TykTechnologies/redis-cluster-operator/internal/redisutil" 7 | ) 8 | 9 | func Test_buildClustersLists(t *testing.T) { 10 | // In the test below, we cannot directly use initialize redisutil.NodeSlice in redisutil.NodeInfos, this is a go vet issue: https://github.com/golang/go/issues/9171 11 | ip1 := redisutil.Nodes{{IP: "ip1", Port: "1234"}} 12 | ip2 := redisutil.Nodes{{IP: "ip2", Port: "1234"}} 13 | ip56 := redisutil.Nodes{{IP: "ip5", Port: "1234"}, {IP: "ip6", Port: "1234"}} 14 | ip64 := redisutil.Nodes{{IP: "ip6", Port: "1234"}, {IP: "ip4", Port: "1234"}} 15 | ip54 := redisutil.Nodes{{IP: "ip5", Port: "1234"}, {IP: "ip4", Port: "1234"}} 16 | // end of workaround 17 | testCases := []struct { 18 | input *redisutil.ClusterInfos 19 | output []cluster 20 | }{ //several partilly different cannot happen, so not tested 21 | { // empty 22 | input: &redisutil.ClusterInfos{Infos: map[string]*redisutil.NodeInfos{}, Status: redisutil.ClusterInfosConsistent}, 23 | output: []cluster{}, 24 | }, 25 | { // one node 26 | input: &redisutil.ClusterInfos{Infos: map[string]*redisutil.NodeInfos{"ip1:1234": {Node: &redisutil.Node{IP: "ip1", Port: "1234"}, Friends: redisutil.Nodes{}}}, Status: redisutil.ClusterInfosConsistent}, 27 | output: []cluster{{"ip1:1234"}}, 28 | }, 29 | { // no discrepency 30 | input: &redisutil.ClusterInfos{ 31 | Infos: map[string]*redisutil.NodeInfos{ 32 | "ip1:1234": {Node: &redisutil.Node{IP: "ip1", Port: "1234"}, Friends: ip2}, 33 | "ip2:1234": {Node: &redisutil.Node{IP: "ip2", Port: "1234"}, Friends: ip1}, 34 | }, 35 | Status: redisutil.ClusterInfosConsistent, 36 | }, 37 | output: []cluster{{"ip1:1234", "ip2:1234"}}, 38 | }, 39 | { // several decorelated 40 | input: &redisutil.ClusterInfos{ 41 | Infos: map[string]*redisutil.NodeInfos{ 42 | "ip1:1234": {Node: &redisutil.Node{IP: "ip1", Port: "1234"}, Friends: ip2}, 43 | "ip2:1234": {Node: &redisutil.Node{IP: "ip2", Port: "1234"}, Friends: ip1}, 44 | "ip3:1234": {Node: &redisutil.Node{IP: "ip3", Port: "1234"}, Friends: redisutil.Nodes{}}, 45 | "ip4:1234": {Node: &redisutil.Node{IP: "ip4", Port: "1234"}, Friends: ip56}, 46 | "ip5:1234": {Node: &redisutil.Node{IP: "ip5", Port: "1234"}, Friends: ip64}, 47 | "ip6:1234": {Node: &redisutil.Node{IP: "ip6", Port: "1234"}, Friends: ip54}, 48 | }, 49 | Status: redisutil.ClusterInfosInconsistent, 50 | }, 51 | output: []cluster{{"ip1:1234", "ip2:1234"}, {"ip3:1234"}, {"ip4:1234", "ip5:1234", "ip6:1234"}}, 52 | }, 53 | { // empty ignored 54 | input: &redisutil.ClusterInfos{ 55 | Infos: map[string]*redisutil.NodeInfos{ 56 | "ip1:1234": {Node: &redisutil.Node{IP: "ip1", Port: "1234"}, Friends: ip2}, 57 | "ip2:1234": {Node: &redisutil.Node{IP: "ip2", Port: "1234"}, Friends: ip1}, 58 | "ip3:1234": nil, 59 | }, 60 | Status: redisutil.ClusterInfosInconsistent, 61 | }, 62 | output: []cluster{{"ip1:1234", "ip2:1234"}}, 63 | }, 64 | } 65 | 66 | for i, tc := range testCases { 67 | output := buildClustersLists(tc.input) 68 | // because we work with map, order might not be conserved 69 | if !compareClusters(output, tc.output) { 70 | t.Errorf("[Case %d] Unexpected result for buildClustersLists, expected %v, got %v", i, tc.output, output) 71 | } 72 | } 73 | } 74 | 75 | func compareClusters(c1, c2 []cluster) bool { 76 | if len(c1) != len(c2) { 77 | return false 78 | } 79 | 80 | for _, c1elem := range c2 { 81 | found := false 82 | for _, c2elem := range c1 { 83 | if compareCluster(c1elem, c2elem) { 84 | found = true 85 | break 86 | } 87 | } 88 | if !found { 89 | return false 90 | } 91 | } 92 | 93 | return true 94 | } 95 | 96 | func compareCluster(c1, c2 cluster) bool { 97 | if len(c1) != len(c2) { 98 | return false 99 | } 100 | for _, c1elem := range c2 { 101 | found := false 102 | for _, c2elem := range c1 { 103 | if c1elem == c2elem { 104 | found = true 105 | break 106 | } 107 | } 108 | if !found { 109 | return false 110 | } 111 | } 112 | 113 | return true 114 | } 115 | -------------------------------------------------------------------------------- /internal/heal/failednodes.go: -------------------------------------------------------------------------------- 1 | package heal 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/util/errors" 5 | 6 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 7 | "github.com/TykTechnologies/redis-cluster-operator/internal/redisutil" 8 | ) 9 | 10 | // FixFailedNodes fix failed nodes: in some cases (cluster without enough master after crash or scale down), some nodes may still know about fail nodes 11 | func (c *CheckAndHeal) FixFailedNodes(cluster *redisv1alpha1.DistributedRedisCluster, infos *redisutil.ClusterInfos, admin redisutil.IAdmin) (bool, error) { 12 | forgetSet := listGhostNodes(cluster, infos) 13 | var errs []error 14 | doneAnAction := false 15 | for id := range forgetSet { 16 | doneAnAction = true 17 | c.Logger.Info("[FixFailedNodes] Forgetting failed node, this command might fail, this is not an error", "node", id) 18 | if !c.DryRun { 19 | c.Logger.Info("[FixFailedNodes] try to forget node", "nodeId", id) 20 | if err := admin.ForgetNode(id); err != nil { 21 | errs = append(errs, err) 22 | } 23 | } 24 | } 25 | 26 | return doneAnAction, errors.NewAggregate(errs) 27 | } 28 | 29 | // listGhostNodes: A Ghost node is a node still known by some redis node but which doesn't exist anymore, 30 | // meaning it is failed, and pod not in kubernetes, or without targetable IP 31 | func listGhostNodes(cluster *redisv1alpha1.DistributedRedisCluster, infos *redisutil.ClusterInfos) map[string]bool { 32 | ghostNodesSet := map[string]bool{} 33 | if infos == nil || infos.Infos == nil { 34 | return ghostNodesSet 35 | } 36 | for _, nodeinfos := range infos.Infos { 37 | for _, node := range nodeinfos.Friends { 38 | // only forget it when no more part of kubernetes, or if noaddress 39 | if node.HasStatus(redisutil.NodeStatusNoAddr) { 40 | ghostNodesSet[node.ID] = true 41 | } 42 | if node.HasStatus(redisutil.NodeStatusFail) || node.HasStatus(redisutil.NodeStatusPFail) { 43 | found := false 44 | for _, pod := range cluster.Status.Nodes { 45 | if pod.ID == node.ID { 46 | found = true 47 | } 48 | } 49 | if !found { 50 | ghostNodesSet[node.ID] = true 51 | } 52 | } 53 | } 54 | } 55 | return ghostNodesSet 56 | } 57 | -------------------------------------------------------------------------------- /internal/heal/heal.go: -------------------------------------------------------------------------------- 1 | package heal 2 | 3 | import ( 4 | "github.com/go-logr/logr" 5 | corev1 "k8s.io/api/core/v1" 6 | 7 | "github.com/TykTechnologies/redis-cluster-operator/internal/k8sutil" 8 | ) 9 | 10 | type CheckAndHeal struct { 11 | Logger logr.Logger 12 | PodControl k8sutil.IPodControl 13 | Pods []*corev1.Pod 14 | DryRun bool 15 | } 16 | -------------------------------------------------------------------------------- /internal/heal/terminatingpod.go: -------------------------------------------------------------------------------- 1 | package heal 2 | 3 | import ( 4 | "time" 5 | 6 | "k8s.io/apimachinery/pkg/util/errors" 7 | 8 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 9 | ) 10 | 11 | // FixTerminatingPods used to for the deletion of pod blocked in terminating status. 12 | // in it append the this method will for the deletion of the Pod. 13 | func (c *CheckAndHeal) FixTerminatingPods(cluster *redisv1alpha1.DistributedRedisCluster, maxDuration time.Duration) (bool, error) { 14 | var errs []error 15 | var actionDone bool 16 | 17 | if maxDuration == time.Duration(0) { 18 | return actionDone, nil 19 | } 20 | 21 | now := time.Now() 22 | for _, pod := range c.Pods { 23 | if pod.DeletionTimestamp == nil { 24 | // ignore pod without a deletion timestamp 25 | continue 26 | } 27 | maxTime := pod.DeletionTimestamp.Add(maxDuration) // adding MaxDuration for configuration 28 | if maxTime.Before(now) { 29 | c.Logger.Info("[FixTerminatingPods] found deletion pod", "podName", pod.Name) 30 | actionDone = true 31 | // it means that this pod should already been deleted since a wild 32 | if !c.DryRun { 33 | c.Logger.Info("[FixTerminatingPods] try to delete pod", "podName", pod.Name) 34 | if err := c.PodControl.DeletePodByName(cluster.Namespace, pod.Name); err != nil { 35 | errs = append(errs, err) 36 | } 37 | } 38 | } 39 | } 40 | 41 | return actionDone, errors.NewAggregate(errs) 42 | } 43 | -------------------------------------------------------------------------------- /internal/heal/untrustenodes.go: -------------------------------------------------------------------------------- 1 | package heal 2 | 3 | import ( 4 | corev1 "k8s.io/api/core/v1" 5 | "k8s.io/apimachinery/pkg/util/errors" 6 | 7 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 8 | "github.com/TykTechnologies/redis-cluster-operator/internal/redisutil" 9 | ) 10 | 11 | // FixUntrustedNodes used to remove Nodes that are not trusted by other nodes. It can append when a node 12 | // are removed from the cluster (with the "forget nodes" command) but try to rejoin the cluster. 13 | func (c *CheckAndHeal) FixUntrustedNodes(cluster *redisv1alpha1.DistributedRedisCluster, infos *redisutil.ClusterInfos, admin redisutil.IAdmin) (bool, error) { 14 | untrustedNode := listUntrustedNodes(infos) 15 | var errs []error 16 | doneAnAction := false 17 | 18 | for id, uNode := range untrustedNode { 19 | c.Logger.Info("[FixUntrustedNodes] found untrust node", "node", uNode) 20 | getByIPFunc := func(n *redisutil.Node) bool { 21 | if n.IP == uNode.IP && n.ID != uNode.ID { 22 | return true 23 | } 24 | return false 25 | } 26 | node2, err := infos.GetNodes().GetNodesByFunc(getByIPFunc) 27 | if err != nil && !redisutil.IsNodeNotFoundedError(err) { 28 | c.Logger.Error(err, "error with GetNodesByFunc(getByIPFunc) search function") 29 | errs = append(errs, err) 30 | continue 31 | } 32 | if len(node2) > 0 { 33 | // it means the POD is used by another Redis node ID so we should not delete the pod. 34 | continue 35 | } 36 | exist, reused := checkIfPodNameExistAndIsReused(uNode, c.Pods) 37 | if exist && !reused { 38 | c.Logger.Info("[FixUntrustedNodes] try to delete pod", "podName", uNode.PodName) 39 | if err := c.PodControl.DeletePodByName(cluster.Namespace, uNode.PodName); err != nil { 40 | errs = append(errs, err) 41 | } 42 | } 43 | doneAnAction = true 44 | if !c.DryRun { 45 | c.Logger.Info("[FixUntrustedNodes] try to forget node", "nodeId", id) 46 | if err := admin.ForgetNode(id); err != nil { 47 | errs = append(errs, err) 48 | } 49 | } 50 | } 51 | 52 | return doneAnAction, errors.NewAggregate(errs) 53 | } 54 | 55 | func listUntrustedNodes(infos *redisutil.ClusterInfos) map[string]*redisutil.Node { 56 | untrustedNodes := make(map[string]*redisutil.Node) 57 | if infos == nil || infos.Infos == nil { 58 | return untrustedNodes 59 | } 60 | for _, nodeinfos := range infos.Infos { 61 | for _, node := range nodeinfos.Friends { 62 | if node.HasStatus(redisutil.NodeStatusHandshake) { 63 | if _, found := untrustedNodes[node.ID]; !found { 64 | untrustedNodes[node.ID] = node 65 | } 66 | } 67 | } 68 | } 69 | return untrustedNodes 70 | } 71 | 72 | func checkIfPodNameExistAndIsReused(node *redisutil.Node, podlist []*corev1.Pod) (exist bool, reused bool) { 73 | if node.PodName == "" { 74 | return 75 | } 76 | for _, currentPod := range podlist { 77 | if currentPod.Name == node.PodName { 78 | exist = true 79 | if currentPod.Status.PodIP == node.IP { 80 | // this check is used to see if the Pod name is not use by another RedisNode. 81 | // for that we check the Pod name from the Redis node is not used by another 82 | // Redis node, by comparing the IP of the current Pod with the Pod from the cluster bom. 83 | // if the Pod IP and Name from the redis info is equal to the IP/NAME from the getPod; it 84 | // means that the Pod is still use and the Redis Node is not a ghost 85 | reused = true 86 | break 87 | } 88 | } 89 | } 90 | return 91 | } 92 | -------------------------------------------------------------------------------- /internal/k8sutil/batchcronjob.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | 6 | batchv1 "k8s.io/api/batch/v1" 7 | "k8s.io/apimachinery/pkg/types" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | ) 10 | 11 | // ICronJobControl define the interface that uses to create, update, and delete CronJobs. 12 | type ICronJobControl interface { 13 | CreateCronJob(*batchv1.CronJob) error 14 | UpdateCronJob(*batchv1.CronJob) error 15 | DeleteCronJob(*batchv1.CronJob) error 16 | GetCronJob(namespace, name string) (*batchv1.CronJob, error) 17 | ListCronJobByLabels(namespace string, labs client.MatchingLabels) (*batchv1.CronJobList, error) 18 | } 19 | 20 | type CronJobController struct { 21 | client client.Client 22 | } 23 | 24 | // NewCronJobController creates a concrete implementation of the 25 | // IJobControl. 26 | func NewCronJobController(client client.Client) ICronJobControl { 27 | return &CronJobController{client: client} 28 | } 29 | 30 | func (c *CronJobController) CreateCronJob(cronjob *batchv1.CronJob) error { 31 | return c.client.Create(context.TODO(), cronjob) 32 | } 33 | 34 | func (c CronJobController) UpdateCronJob(cronjob *batchv1.CronJob) error { 35 | return c.client.Update(context.TODO(), cronjob) 36 | } 37 | 38 | func (c CronJobController) DeleteCronJob(cronjob *batchv1.CronJob) error { 39 | return c.client.Delete(context.TODO(), cronjob) 40 | } 41 | 42 | func (c CronJobController) GetCronJob(namespace, name string) (*batchv1.CronJob, error) { 43 | cronjob := &batchv1.CronJob{} 44 | err := c.client.Get(context.TODO(), types.NamespacedName{ 45 | Name: name, 46 | Namespace: namespace, 47 | }, cronjob) 48 | return cronjob, err 49 | } 50 | 51 | func (c CronJobController) ListCronJobByLabels(namespace string, labels client.MatchingLabels) (*batchv1.CronJobList, error) { 52 | cronjobList := &batchv1.CronJobList{} 53 | opts := []client.ListOption{ 54 | client.InNamespace(namespace), 55 | labels, 56 | } 57 | err := c.client.List(context.TODO(), cronjobList, opts...) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | return cronjobList, nil 63 | } 64 | -------------------------------------------------------------------------------- /internal/k8sutil/batchjob.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | 6 | batchv1 "k8s.io/api/batch/v1" 7 | "k8s.io/apimachinery/pkg/types" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | ) 10 | 11 | // IJobControl definej the interface that usej to create, update, and delete Jobs. 12 | type IJobControl interface { 13 | CreateJob(*batchv1.Job) error 14 | UpdateJob(*batchv1.Job) error 15 | DeleteJob(*batchv1.Job) error 16 | GetJob(namespace, name string) (*batchv1.Job, error) 17 | ListJobByLabels(namespace string, labs client.MatchingLabels) (*batchv1.JobList, error) 18 | } 19 | 20 | type JobController struct { 21 | client client.Client 22 | } 23 | 24 | // NewJobController creates a concrete implementation of the 25 | // IJobControl. 26 | func NewJobController(client client.Client) IJobControl { 27 | return &JobController{client: client} 28 | } 29 | 30 | // CreateJob implement the IJobControl.Interface. 31 | func (j *JobController) CreateJob(job *batchv1.Job) error { 32 | return j.client.Create(context.TODO(), job) 33 | } 34 | 35 | // UpdateJob implement the IJobControl.Interface. 36 | func (j *JobController) UpdateJob(job *batchv1.Job) error { 37 | return j.client.Update(context.TODO(), job) 38 | } 39 | 40 | // DeleteJob implement the IJobControl.Interface. 41 | func (j *JobController) DeleteJob(job *batchv1.Job) error { 42 | return j.client.Delete(context.TODO(), job) 43 | } 44 | 45 | // GetJob implement the IJobControl.Interface. 46 | func (j *JobController) GetJob(namespace, name string) (*batchv1.Job, error) { 47 | job := &batchv1.Job{} 48 | err := j.client.Get(context.TODO(), types.NamespacedName{ 49 | Name: name, 50 | Namespace: namespace, 51 | }, job) 52 | return job, err 53 | } 54 | 55 | func (j *JobController) ListJobByLabels(namespace string, labs client.MatchingLabels) (*batchv1.JobList, error) { 56 | jobList := &batchv1.JobList{} 57 | opts := []client.ListOption{ 58 | client.InNamespace(namespace), 59 | labs, 60 | } 61 | err := j.client.List(context.TODO(), jobList, opts...) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | return jobList, nil 67 | } 68 | -------------------------------------------------------------------------------- /internal/k8sutil/configmap.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/apimachinery/pkg/types" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | ) 10 | 11 | // IConfigMapControl defines the interface that uses to create, update, and delete ConfigMaps. 12 | type IConfigMapControl interface { 13 | // CreateConfigMap creates a ConfigMap in a DistributedRedisCluster. 14 | CreateConfigMap(*corev1.ConfigMap) error 15 | // UpdateConfigMap updates a ConfigMap in a DistributedRedisCluster. 16 | UpdateConfigMap(*corev1.ConfigMap) error 17 | // DeleteConfigMap deletes a ConfigMap in a DistributedRedisCluster. 18 | DeleteConfigMap(*corev1.ConfigMap) error 19 | // GetConfigMap get ConfigMap in a DistributedRedisCluster. 20 | GetConfigMap(namespace, name string) (*corev1.ConfigMap, error) 21 | } 22 | 23 | type ConfigMapController struct { 24 | client client.Client 25 | } 26 | 27 | // NewConfigMapController creates a concrete implementation of the 28 | // IConfigMapControl. 29 | func NewConfigMapController(client client.Client) IConfigMapControl { 30 | return &ConfigMapController{client: client} 31 | } 32 | 33 | // CreateConfigMap implement the IConfigMapControl.Interface. 34 | func (s *ConfigMapController) CreateConfigMap(cm *corev1.ConfigMap) error { 35 | return s.client.Create(context.TODO(), cm) 36 | } 37 | 38 | // UpdateConfigMap implement the IConfigMapControl.Interface. 39 | func (s *ConfigMapController) UpdateConfigMap(cm *corev1.ConfigMap) error { 40 | return s.client.Update(context.TODO(), cm) 41 | } 42 | 43 | // DeleteConfigMap implement the IConfigMapControl.Interface. 44 | func (s *ConfigMapController) DeleteConfigMap(cm *corev1.ConfigMap) error { 45 | return s.client.Delete(context.TODO(), cm) 46 | } 47 | 48 | // GetConfigMap implement the IConfigMapControl.Interface. 49 | func (s *ConfigMapController) GetConfigMap(namespace, name string) (*corev1.ConfigMap, error) { 50 | cm := &corev1.ConfigMap{} 51 | err := s.client.Get(context.TODO(), types.NamespacedName{ 52 | Name: name, 53 | Namespace: namespace, 54 | }, cm) 55 | return cm, err 56 | } 57 | -------------------------------------------------------------------------------- /internal/k8sutil/customresource.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | 6 | "k8s.io/apimachinery/pkg/types" 7 | 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | 10 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 11 | ) 12 | 13 | // ICustomResource defines the interface that uses to update cr status 14 | type ICustomResource interface { 15 | // UpdateCRStatus update the RedisCluster status 16 | UpdateCRStatus(client.Object) error 17 | UpdateCR(client.Object) error 18 | GetRedisClusterBackup(namespace, name string) (*redisv1alpha1.RedisClusterBackup, error) 19 | GetDistributedRedisCluster(namespace, name string) (*redisv1alpha1.DistributedRedisCluster, error) 20 | } 21 | 22 | type clusterControl struct { 23 | client client.Client 24 | } 25 | 26 | // NewCRControl creates a concrete implementation of the 27 | // ICustomResource. 28 | func NewCRControl(client client.Client) ICustomResource { 29 | return &clusterControl{client: client} 30 | } 31 | 32 | func (c *clusterControl) UpdateCRStatus(obj client.Object) error { 33 | return c.client.Status().Update(context.TODO(), obj) 34 | } 35 | 36 | func (c *clusterControl) UpdateCR(obj client.Object) error { 37 | return c.client.Update(context.TODO(), obj) 38 | } 39 | 40 | func (c *clusterControl) GetRedisClusterBackup(namespace, name string) (*redisv1alpha1.RedisClusterBackup, error) { 41 | backup := &redisv1alpha1.RedisClusterBackup{} 42 | if err := c.client.Get(context.TODO(), types.NamespacedName{ 43 | Name: name, 44 | Namespace: namespace, 45 | }, backup); err != nil { 46 | return nil, err 47 | } 48 | return backup, nil 49 | } 50 | 51 | func (c *clusterControl) GetDistributedRedisCluster(namespace, name string) (*redisv1alpha1.DistributedRedisCluster, error) { 52 | drc := &redisv1alpha1.DistributedRedisCluster{} 53 | if err := c.client.Get(context.TODO(), types.NamespacedName{ 54 | Name: name, 55 | Namespace: namespace, 56 | }, drc); err != nil { 57 | return nil, err 58 | } 59 | return drc, nil 60 | } 61 | -------------------------------------------------------------------------------- /internal/k8sutil/pod.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/apimachinery/pkg/types" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | ) 10 | 11 | // IPodControl defines the interface that uses to create, update, and delete Pods. 12 | type IPodControl interface { 13 | // CreatePod creates a Pod in a DistributedRedisCluster. 14 | CreatePod(*corev1.Pod) error 15 | // UpdatePod updates a Pod in a DistributedRedisCluster. 16 | UpdatePod(*corev1.Pod) error 17 | // DeletePod deletes a Pod in a DistributedRedisCluster. 18 | DeletePod(*corev1.Pod) error 19 | DeletePodByName(namespace, name string) error 20 | // GetPod get Pod in a DistributedRedisCluster. 21 | GetPod(namespace, name string) (*corev1.Pod, error) 22 | } 23 | 24 | type PodController struct { 25 | client client.Client 26 | } 27 | 28 | // NewPodController creates a concrete implementation of the 29 | // IPodControl. 30 | func NewPodController(client client.Client) IPodControl { 31 | return &PodController{client: client} 32 | } 33 | 34 | // CreatePod implement the IPodControl.Interface. 35 | func (p *PodController) CreatePod(pod *corev1.Pod) error { 36 | return p.client.Create(context.TODO(), pod) 37 | } 38 | 39 | // UpdatePod implement the IPodControl.Interface. 40 | func (p *PodController) UpdatePod(pod *corev1.Pod) error { 41 | return p.client.Update(context.TODO(), pod) 42 | } 43 | 44 | // DeletePod implement the IPodControl.Interface. 45 | func (p *PodController) DeletePod(pod *corev1.Pod) error { 46 | return p.client.Delete(context.TODO(), pod) 47 | } 48 | 49 | // DeletePodByName implement the IPodControl.Interface. 50 | func (p *PodController) DeletePodByName(namespace, name string) error { 51 | pod, err := p.GetPod(namespace, name) 52 | if err != nil { 53 | return err 54 | } 55 | return p.client.Delete(context.TODO(), pod) 56 | } 57 | 58 | // GetPod implement the IPodControl.Interface. 59 | func (p *PodController) GetPod(namespace, name string) (*corev1.Pod, error) { 60 | pod := &corev1.Pod{} 61 | err := p.client.Get(context.TODO(), types.NamespacedName{ 62 | Name: name, 63 | Namespace: namespace, 64 | }, pod) 65 | return pod, err 66 | } 67 | -------------------------------------------------------------------------------- /internal/k8sutil/poddisruptionbudget.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | 6 | policyv11 "k8s.io/api/policy/v1" 7 | "k8s.io/apimachinery/pkg/types" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | ) 10 | 11 | // IPodDisruptionBudgetControl defines the interface that uses to create, update, and delete PodDisruptionBudgets. 12 | type IPodDisruptionBudgetControl interface { 13 | // CreatePodDisruptionBudget creates a PodDisruptionBudget in a DistributedRedisCluster. 14 | CreatePodDisruptionBudget(*policyv11.PodDisruptionBudget) error 15 | // UpdatePodDisruptionBudget updates a PodDisruptionBudget in a DistributedRedisCluster. 16 | UpdatePodDisruptionBudget(*policyv11.PodDisruptionBudget) error 17 | // DeletePodDisruptionBudget deletes a PodDisruptionBudget in a DistributedRedisCluster. 18 | DeletePodDisruptionBudget(*policyv11.PodDisruptionBudget) error 19 | DeletePodDisruptionBudgetByName(namespace, name string) error 20 | // GetPodDisruptionBudget get PodDisruptionBudget in a DistributedRedisCluster. 21 | GetPodDisruptionBudget(namespace, name string) (*policyv11.PodDisruptionBudget, error) 22 | } 23 | 24 | type PodDisruptionBudgetController struct { 25 | client client.Client 26 | } 27 | 28 | // NewPodDisruptionBudgetController creates a concrete implementation of the 29 | // IPodDisruptionBudgetControl. 30 | func NewPodDisruptionBudgetController(client client.Client) IPodDisruptionBudgetControl { 31 | return &PodDisruptionBudgetController{client: client} 32 | } 33 | 34 | // CreatePodDisruptionBudget implement the IPodDisruptionBudgetControl.Interface. 35 | func (s *PodDisruptionBudgetController) CreatePodDisruptionBudget(pb *policyv11.PodDisruptionBudget) error { 36 | return s.client.Create(context.TODO(), pb) 37 | } 38 | 39 | // UpdatePodDisruptionBudget implement the IPodDisruptionBudgetControl.Interface. 40 | func (s *PodDisruptionBudgetController) UpdatePodDisruptionBudget(pb *policyv11.PodDisruptionBudget) error { 41 | return s.client.Update(context.TODO(), pb) 42 | } 43 | 44 | // DeletePodDisruptionBudget implement the IPodDisruptionBudgetControl.Interface. 45 | func (s *PodDisruptionBudgetController) DeletePodDisruptionBudget(pb *policyv11.PodDisruptionBudget) error { 46 | return s.client.Delete(context.TODO(), pb) 47 | } 48 | 49 | func (s *PodDisruptionBudgetController) DeletePodDisruptionBudgetByName(namespace, name string) error { 50 | pdb, err := s.GetPodDisruptionBudget(namespace, name) 51 | if err != nil { 52 | return err 53 | } 54 | return s.DeletePodDisruptionBudget(pdb) 55 | } 56 | 57 | // GetPodDisruptionBudget implement the IPodDisruptionBudgetControl.Interface. 58 | func (s *PodDisruptionBudgetController) GetPodDisruptionBudget(namespace, name string) (*policyv11.PodDisruptionBudget, error) { 59 | pb := &policyv11.PodDisruptionBudget{} 60 | err := s.client.Get(context.TODO(), types.NamespacedName{ 61 | Name: name, 62 | Namespace: namespace, 63 | }, pb) 64 | return pb, err 65 | } 66 | -------------------------------------------------------------------------------- /internal/k8sutil/pvc.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/apimachinery/pkg/types" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | ) 10 | 11 | // IPvcControl defines the interface that uses to create, update, and delete PersistentVolumeClaim. 12 | type IPvcControl interface { 13 | DeletePvc(claim *corev1.PersistentVolumeClaim) error 14 | DeletePvcByLabels(namespace string, labels map[string]string) error 15 | GetPvc(namespace, name string) (*corev1.PersistentVolumeClaim, error) 16 | } 17 | 18 | type pvcController struct { 19 | client client.Client 20 | } 21 | 22 | // NewPvcController creates a concrete implementation of the 23 | // IPvcControl. 24 | func NewPvcController(client client.Client) IPvcControl { 25 | return &pvcController{client: client} 26 | } 27 | 28 | // DeletePvc implement the IPvcControl.Interface. 29 | func (s *pvcController) DeletePvc(pvc *corev1.PersistentVolumeClaim) error { 30 | return s.client.Delete(context.TODO(), pvc) 31 | } 32 | 33 | func (s *pvcController) DeletePvcByLabels(namespace string, labels map[string]string) error { 34 | foundPvcs := &corev1.PersistentVolumeClaimList{} 35 | err := s.client.List(context.TODO(), foundPvcs, client.InNamespace(namespace), client.MatchingLabels(labels)) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | for _, pvc := range foundPvcs.Items { 41 | if err := s.client.Delete(context.TODO(), &pvc); err != nil { 42 | return err 43 | } 44 | } 45 | return nil 46 | } 47 | 48 | // GetPvc implement the IPvcControl.Interface. 49 | func (s *pvcController) GetPvc(namespace, name string) (*corev1.PersistentVolumeClaim, error) { 50 | pvc := &corev1.PersistentVolumeClaim{} 51 | err := s.client.Get(context.TODO(), types.NamespacedName{ 52 | Name: name, 53 | Namespace: namespace, 54 | }, pvc) 55 | return pvc, err 56 | } 57 | -------------------------------------------------------------------------------- /internal/k8sutil/service.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/apimachinery/pkg/types" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | ) 10 | 11 | // IServiceControl defines the interface that uses to create, update, and delete Services. 12 | type IServiceControl interface { 13 | // CreateService creates a Service in a DistributedRedisCluster. 14 | CreateService(*corev1.Service) error 15 | // UpdateService updates a Service in a DistributedRedisCluster. 16 | UpdateService(*corev1.Service) error 17 | // DeleteService deletes a Service in a DistributedRedisCluster. 18 | DeleteService(*corev1.Service) error 19 | DeleteServiceByName(namespace, name string) error 20 | // GetService get Service in a DistributedRedisCluster. 21 | GetService(namespace, name string) (*corev1.Service, error) 22 | } 23 | 24 | type serviceController struct { 25 | client client.Client 26 | } 27 | 28 | // NewServiceController creates a concrete implementation of the 29 | // IServiceControl. 30 | func NewServiceController(client client.Client) IServiceControl { 31 | return &serviceController{client: client} 32 | } 33 | 34 | // CreateService implement the IServiceControl.Interface. 35 | func (s *serviceController) CreateService(svc *corev1.Service) error { 36 | return s.client.Create(context.TODO(), svc) 37 | } 38 | 39 | // UpdateService implement the IServiceControl.Interface. 40 | func (s *serviceController) UpdateService(svc *corev1.Service) error { 41 | return s.client.Update(context.TODO(), svc) 42 | } 43 | 44 | // DeleteService implement the IServiceControl.Interface. 45 | func (s *serviceController) DeleteService(svc *corev1.Service) error { 46 | return s.client.Delete(context.TODO(), svc) 47 | } 48 | 49 | func (s *serviceController) DeleteServiceByName(namespace, name string) error { 50 | svc, err := s.GetService(namespace, name) 51 | if err != nil { 52 | return err 53 | } 54 | return s.DeleteService(svc) 55 | } 56 | 57 | // GetService implement the IServiceControl.Interface. 58 | func (s *serviceController) GetService(namespace, name string) (*corev1.Service, error) { 59 | svc := &corev1.Service{} 60 | err := s.client.Get(context.TODO(), types.NamespacedName{ 61 | Name: name, 62 | Namespace: namespace, 63 | }, svc) 64 | return svc, err 65 | } 66 | -------------------------------------------------------------------------------- /internal/k8sutil/statefulset.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | 6 | appsv1 "k8s.io/api/apps/v1" 7 | corev1 "k8s.io/api/core/v1" 8 | "k8s.io/apimachinery/pkg/types" 9 | "sigs.k8s.io/controller-runtime/pkg/client" 10 | ) 11 | 12 | // IStatefulSetControl defines the interface that uses to create, update, and delete StatefulSets. 13 | type IStatefulSetControl interface { 14 | // CreateStatefulSet creates a StatefulSet in a DistributedRedisCluster. 15 | CreateStatefulSet(*appsv1.StatefulSet) error 16 | // UpdateStatefulSet updates a StatefulSet in a DistributedRedisCluster. 17 | UpdateStatefulSet(*appsv1.StatefulSet) error 18 | // DeleteStatefulSet deletes a StatefulSet in a DistributedRedisCluster. 19 | DeleteStatefulSet(*appsv1.StatefulSet) error 20 | DeleteStatefulSetByName(namespace, name string) error 21 | // GetStatefulSet get StatefulSet in a DistributedRedisCluster. 22 | GetStatefulSet(namespace, name string) (*appsv1.StatefulSet, error) 23 | ListStatefulSetByLabels(namespace string, labels map[string]string) (*appsv1.StatefulSetList, error) 24 | // GetStatefulSetPods will retrieve the pods managed by a given StatefulSet. 25 | GetStatefulSetPods(namespace, name string) (*corev1.PodList, error) 26 | GetStatefulSetPodsByLabels(namespace string, labels map[string]string) (*corev1.PodList, error) 27 | } 28 | 29 | type stateFulSetController struct { 30 | client client.Client 31 | } 32 | 33 | // NewStatefulSetController creates a concrete implementation of the 34 | // IStatefulSetControl. 35 | func NewStatefulSetController(client client.Client) IStatefulSetControl { 36 | return &stateFulSetController{client: client} 37 | } 38 | 39 | // CreateStatefulSet implement the IStatefulSetControl.Interface. 40 | func (s *stateFulSetController) CreateStatefulSet(ss *appsv1.StatefulSet) error { 41 | return s.client.Create(context.TODO(), ss) 42 | } 43 | 44 | // UpdateStatefulSet implement the IStatefulSetControl.Interface. 45 | func (s *stateFulSetController) UpdateStatefulSet(ss *appsv1.StatefulSet) error { 46 | return s.client.Update(context.TODO(), ss) 47 | } 48 | 49 | // DeleteStatefulSet implement the IStatefulSetControl.Interface. 50 | func (s *stateFulSetController) DeleteStatefulSet(ss *appsv1.StatefulSet) error { 51 | return s.client.Delete(context.TODO(), ss) 52 | } 53 | 54 | func (s *stateFulSetController) DeleteStatefulSetByName(namespace, name string) error { 55 | sts, err := s.GetStatefulSet(namespace, name) 56 | if err != nil { 57 | return err 58 | } 59 | return s.DeleteStatefulSet(sts) 60 | } 61 | 62 | // GetStatefulSet implement the IStatefulSetControl.Interface. 63 | func (s *stateFulSetController) GetStatefulSet(namespace, name string) (*appsv1.StatefulSet, error) { 64 | statefulSet := &appsv1.StatefulSet{} 65 | err := s.client.Get(context.TODO(), types.NamespacedName{ 66 | Name: name, 67 | Namespace: namespace, 68 | }, statefulSet) 69 | return statefulSet, err 70 | } 71 | 72 | // GetStatefulSetPods implement the IStatefulSetControl.Interface. 73 | func (s *stateFulSetController) GetStatefulSetPods(namespace, name string) (*corev1.PodList, error) { 74 | statefulSet, err := s.GetStatefulSet(namespace, name) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | match := make(client.MatchingLabels) 80 | for k, v := range statefulSet.Spec.Selector.MatchLabels { 81 | match[k] = v 82 | } 83 | foundPods := &corev1.PodList{} 84 | err = s.client.List(context.TODO(), foundPods, client.InNamespace(namespace), match) 85 | return foundPods, err 86 | } 87 | 88 | // GetStatefulSetPodsByLabels implement the IStatefulSetControl.Interface. 89 | func (s *stateFulSetController) GetStatefulSetPodsByLabels(namespace string, labels map[string]string) (*corev1.PodList, error) { 90 | foundPods := &corev1.PodList{} 91 | err := s.client.List(context.TODO(), foundPods, client.InNamespace(namespace), client.MatchingLabels(labels)) 92 | return foundPods, err 93 | } 94 | 95 | func (s *stateFulSetController) ListStatefulSetByLabels(namespace string, labels map[string]string) (*appsv1.StatefulSetList, error) { 96 | foundSts := &appsv1.StatefulSetList{} 97 | err := s.client.List(context.TODO(), foundSts, client.InNamespace(namespace), client.MatchingLabels(labels)) 98 | return foundSts, err 99 | } 100 | -------------------------------------------------------------------------------- /internal/k8sutil/util.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-logr/logr" 7 | corev1 "k8s.io/api/core/v1" 8 | "k8s.io/apimachinery/pkg/api/errors" 9 | kerr "k8s.io/apimachinery/pkg/api/errors" 10 | "k8s.io/apimachinery/pkg/types" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | ) 13 | 14 | func IsRequestRetryable(err error) bool { 15 | return kerr.IsServiceUnavailable(err) || 16 | kerr.IsTimeout(err) || 17 | kerr.IsServerTimeout(err) || 18 | kerr.IsTooManyRequests(err) 19 | } 20 | 21 | func CreateSecret(client client.Client, secret *corev1.Secret, logger logr.Logger) error { 22 | ctx := context.TODO() 23 | s := &corev1.Secret{} 24 | err := client.Get(ctx, types.NamespacedName{ 25 | Namespace: secret.Namespace, 26 | Name: secret.Name, 27 | }, s) 28 | if err != nil { 29 | if errors.IsNotFound(err) { 30 | logger.WithValues("Secret.Namespace", secret.Namespace, "Secret.Name", secret.Name). 31 | Info("creating a new secret") 32 | return client.Create(ctx, secret) 33 | } 34 | } 35 | return err 36 | } 37 | -------------------------------------------------------------------------------- /internal/manager/checker.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "fmt" 5 | 6 | appsv1 "k8s.io/api/apps/v1" 7 | "sigs.k8s.io/controller-runtime/pkg/client" 8 | 9 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 10 | "github.com/TykTechnologies/redis-cluster-operator/internal/k8sutil" 11 | "github.com/TykTechnologies/redis-cluster-operator/internal/resources/statefulsets" 12 | ) 13 | 14 | type ICheck interface { 15 | CheckRedisNodeNum(*redisv1alpha1.DistributedRedisCluster) error 16 | //CheckRedisMasterNum(*redisv1alpha1.DistributedRedisCluster) error 17 | } 18 | 19 | type realCheck struct { 20 | statefulSetClient k8sutil.IStatefulSetControl 21 | } 22 | 23 | func NewCheck(client client.Client) ICheck { 24 | return &realCheck{ 25 | statefulSetClient: k8sutil.NewStatefulSetController(client), 26 | } 27 | } 28 | 29 | func (c *realCheck) CheckRedisNodeNum(cluster *redisv1alpha1.DistributedRedisCluster) error { 30 | for i := 0; i < int(cluster.Spec.MasterSize); i++ { 31 | name := statefulsets.ClusterStatefulSetName(cluster.Name, i) 32 | expectNodeNum := cluster.Spec.ClusterReplicas + 1 33 | ss, err := c.statefulSetClient.GetStatefulSet(cluster.Namespace, name) 34 | if err != nil { 35 | return err 36 | } 37 | if err := c.checkRedisNodeNum(expectNodeNum, ss); err != nil { 38 | return err 39 | } 40 | } 41 | 42 | return nil 43 | } 44 | 45 | func (c *realCheck) checkRedisNodeNum(expectNodeNum int32, ss *appsv1.StatefulSet) error { 46 | if expectNodeNum != *ss.Spec.Replicas { 47 | return fmt.Errorf("number of redis pods is different from specification") 48 | } 49 | if expectNodeNum != ss.Status.ReadyReplicas { 50 | return fmt.Errorf("redis pods are not all ready") 51 | } 52 | if expectNodeNum != ss.Status.CurrentReplicas { 53 | return fmt.Errorf("redis pods need to be updated") 54 | } 55 | 56 | return nil 57 | } 58 | 59 | func (c *realCheck) CheckRedisMasterNum(cluster *redisv1alpha1.DistributedRedisCluster) error { 60 | if cluster.Spec.MasterSize != cluster.Status.NumberOfMaster { 61 | return fmt.Errorf("number of redis master different from specification") 62 | } 63 | return nil 64 | } 65 | 66 | // 67 | //func (c *realCheck) CheckRedisClusterIsEmpty(cluster *redisv1alpha1.DistributedRedisCluster, admin redisutil.IAdmin) (bool, error) { 68 | // 69 | //} 70 | -------------------------------------------------------------------------------- /internal/manager/ensurer_test.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/go-logr/logr" 7 | logf "sigs.k8s.io/controller-runtime/pkg/log" 8 | ) 9 | 10 | var log = logf.Log.WithName("test") 11 | 12 | func Test_isRedisConfChanged(t *testing.T) { 13 | type args struct { 14 | confInCm string 15 | currentConf map[string]string 16 | log logr.Logger 17 | } 18 | tests := []struct { 19 | name string 20 | args args 21 | want bool 22 | }{ 23 | { 24 | name: "should false", 25 | args: args{ 26 | confInCm: `appendfsync everysec 27 | appendonly yes 28 | auto-aof-rewrite-min-size 67108864 29 | save 900 1 300 10`, 30 | currentConf: map[string]string{ 31 | "appendfsync": "everysec", 32 | "appendonly": "yes", 33 | "auto-aof-rewrite-min-size": "67108864", 34 | "save": "900 1 300 10", 35 | }, 36 | log: log, 37 | }, 38 | want: false, 39 | }, 40 | { 41 | name: "should false with newline", 42 | args: args{ 43 | confInCm: `appendfsync everysec 44 | appendonly yes 45 | auto-aof-rewrite-min-size 67108864 46 | save 900 1 300 10 47 | `, 48 | currentConf: map[string]string{ 49 | "appendfsync": "everysec", 50 | "appendonly": "yes", 51 | "auto-aof-rewrite-min-size": "67108864", 52 | "save": "900 1 300 10", 53 | }, 54 | log: log, 55 | }, 56 | want: false, 57 | }, 58 | { 59 | name: "should true, compare value", 60 | args: args{ 61 | confInCm: `appendfsync everysec 62 | appendonly yes 63 | auto-aof-rewrite-min-size 6710886 64 | save 900 1 300 10 65 | `, 66 | currentConf: map[string]string{ 67 | "appendfsync": "everysec", 68 | "appendonly": "yes", 69 | "auto-aof-rewrite-min-size": "67108864", 70 | "save": "900 1 300 10", 71 | }, 72 | log: log, 73 | }, 74 | want: true, 75 | }, 76 | { 77 | name: "should true, add current", 78 | args: args{ 79 | confInCm: `appendfsync everysec 80 | appendonly yes 81 | save 900 1 300 10 82 | `, 83 | currentConf: map[string]string{ 84 | "appendfsync": "everysec", 85 | "appendonly": "yes", 86 | "auto-aof-rewrite-min-size": "67108864", 87 | "save": "900 1 300 10", 88 | }, 89 | log: log, 90 | }, 91 | want: true, 92 | }, 93 | { 94 | name: "should true, del current", 95 | args: args{ 96 | confInCm: `appendfsync everysec 97 | appendonly yes 98 | auto-aof-rewrite-min-size 67108864 99 | save 900 1 300 10 100 | `, 101 | currentConf: map[string]string{ 102 | "appendfsync": "everysec", 103 | "appendonly": "yes", 104 | "save": "900 1 300 10", 105 | }, 106 | log: log, 107 | }, 108 | want: true, 109 | }, 110 | { 111 | name: "should true, compare key", 112 | args: args{ 113 | confInCm: `appendfsync everysec 114 | appendonly yes 115 | save 900 1 300 10 116 | `, 117 | currentConf: map[string]string{ 118 | "appendonly": "yes", 119 | "auto-aof-rewrite-min-size": "67108864", 120 | "save": "900 1 300 10", 121 | }, 122 | log: log, 123 | }, 124 | want: true, 125 | }, 126 | { 127 | name: "should true, compare save", 128 | args: args{ 129 | confInCm: `appendfsync everysec 130 | appendonly yes 131 | auto-aof-rewrite-min-size 67108864 132 | save 900 1 300 10 133 | `, 134 | currentConf: map[string]string{ 135 | "appendfsync": "everysec", 136 | "appendonly": "yes", 137 | "auto-aof-rewrite-min-size": "67108864", 138 | "save": "900 1", 139 | }, 140 | log: log, 141 | }, 142 | want: true, 143 | }, 144 | } 145 | for _, tt := range tests { 146 | t.Run(tt.name, func(t *testing.T) { 147 | if got := isRedisConfChanged(tt.args.confInCm, tt.args.currentConf, tt.args.log); got != tt.want { 148 | t.Errorf("isRedisConfChanged() = %v, want %v", got, tt.want) 149 | } 150 | }) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /internal/manager/healer.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "time" 5 | 6 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 7 | "github.com/TykTechnologies/redis-cluster-operator/internal/heal" 8 | "github.com/TykTechnologies/redis-cluster-operator/internal/redisutil" 9 | ) 10 | 11 | type IHeal interface { 12 | Heal(cluster *redisv1alpha1.DistributedRedisCluster, infos *redisutil.ClusterInfos, admin redisutil.IAdmin) (bool, error) 13 | FixTerminatingPods(cluster *redisv1alpha1.DistributedRedisCluster, maxDuration time.Duration) (bool, error) 14 | } 15 | 16 | type realHeal struct { 17 | *heal.CheckAndHeal 18 | } 19 | 20 | func NewHealer(heal *heal.CheckAndHeal) IHeal { 21 | return &realHeal{heal} 22 | } 23 | 24 | func (h *realHeal) Heal(cluster *redisv1alpha1.DistributedRedisCluster, infos *redisutil.ClusterInfos, admin redisutil.IAdmin) (bool, error) { 25 | if actionDone, err := h.FixFailedNodes(cluster, infos, admin); err != nil { 26 | return actionDone, err 27 | } else if actionDone { 28 | return actionDone, nil 29 | } 30 | 31 | if actionDone, err := h.FixUntrustedNodes(cluster, infos, admin); err != nil { 32 | return actionDone, err 33 | } else if actionDone { 34 | return actionDone, nil 35 | } 36 | return false, nil 37 | } 38 | -------------------------------------------------------------------------------- /internal/osm/context/lib.go: -------------------------------------------------------------------------------- 1 | package context 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/spf13/cobra" 11 | "gomodules.xyz/stow" 12 | 13 | /* 14 | Copyright The osm Authors. 15 | 16 | Licensed under the Apache License, Version 2.0 (the "License"); 17 | you may not use this file except in compliance with the License. 18 | You may obtain a copy of the License at 19 | 20 | http://www.apache.org/licenses/LICENSE-2.0 21 | 22 | Unless required by applicable law or agreed to in writing, software 23 | distributed under the License is distributed on an "AS IS" BASIS, 24 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 25 | See the License for the specific language governing permissions and 26 | limitations under the License. 27 | */ 28 | 29 | "io/ioutil" 30 | 31 | yc "gomodules.xyz/encoding/yaml" 32 | "sigs.k8s.io/yaml" 33 | ) 34 | 35 | type Context struct { 36 | Name string `json:"name"` 37 | Provider string `json:"provider"` 38 | Config stow.ConfigMap `json:"config"` 39 | } 40 | 41 | type OSMConfig struct { 42 | Contexts []*Context `json:"contexts"` 43 | CurrentContext string `json:"current-context"` 44 | } 45 | 46 | func GetConfigPath(cmd *cobra.Command) string { 47 | s, err := cmd.Flags().GetString("osmconfig") 48 | if err != nil { 49 | log.Fatalf("error accessing flag osmconfig for command %s: %v", cmd.Name(), err) 50 | } 51 | return s 52 | } 53 | 54 | func LoadConfig(configPath string) (*OSMConfig, error) { 55 | if _, err := os.Stat(configPath); err != nil { 56 | return nil, err 57 | } 58 | err := os.Chmod(configPath, 0600) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | config := &OSMConfig{} 64 | bytes, err := ioutil.ReadFile(configPath) 65 | if err != nil { 66 | return nil, err 67 | } 68 | jsonData, err := yc.ToJSON(bytes) 69 | if err != nil { 70 | return nil, err 71 | } 72 | err = json.Unmarshal(jsonData, config) 73 | return config, err 74 | } 75 | 76 | func (config *OSMConfig) Save(configPath string) error { 77 | data, err := yaml.Marshal(config) 78 | if err != nil { 79 | return err 80 | } 81 | err = os.MkdirAll(filepath.Dir(configPath), 0755) 82 | if err != nil { 83 | return err 84 | } 85 | if err := ioutil.WriteFile(configPath, data, 0600); err != nil { 86 | return err 87 | } 88 | return nil 89 | } 90 | 91 | func (config *OSMConfig) Dial(cliCtx string) (stow.Location, error) { 92 | ctx := config.CurrentContext 93 | if cliCtx != "" { 94 | ctx = cliCtx 95 | } 96 | for _, osmCtx := range config.Contexts { 97 | if osmCtx.Name == ctx { 98 | return stow.Dial(osmCtx.Provider, osmCtx.Config) 99 | } 100 | } 101 | return nil, errors.New("failed to determine context") 102 | } 103 | 104 | func (config *OSMConfig) Context(cliCtx string) (*Context, error) { 105 | ctx := config.CurrentContext 106 | if cliCtx != "" { 107 | ctx = cliCtx 108 | } 109 | for _, osmCtx := range config.Contexts { 110 | if osmCtx.Name == ctx { 111 | return osmCtx, nil 112 | } 113 | } 114 | return nil, errors.New("failed to determine context") 115 | } 116 | -------------------------------------------------------------------------------- /internal/osm/google/config.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | "net/url" 7 | "strings" 8 | 9 | "golang.org/x/net/context" 10 | "golang.org/x/oauth2" 11 | "golang.org/x/oauth2/google" 12 | storage "google.golang.org/api/storage/v1" 13 | 14 | "gomodules.xyz/stow" 15 | ) 16 | 17 | // Kind represents the name of the location/storage type. 18 | const Kind = "google" 19 | 20 | const ( 21 | // The service account json blob 22 | ConfigJSON = "json" 23 | ConfigProjectId = "project_id" 24 | ConfigScopes = "scopes" 25 | ) 26 | 27 | func init() { 28 | validatefn := func(config stow.Config) error { 29 | _, ok := config.Config(ConfigJSON) 30 | if !ok { 31 | return errors.New("missing JSON configuration") 32 | } 33 | 34 | _, ok = config.Config(ConfigProjectId) 35 | if !ok { 36 | return errors.New("missing Project ID") 37 | } 38 | return nil 39 | } 40 | makefn := func(config stow.Config) (stow.Location, error) { 41 | _, ok := config.Config(ConfigJSON) 42 | if !ok { 43 | return nil, errors.New("missing JSON configuration") 44 | } 45 | 46 | _, ok = config.Config(ConfigProjectId) 47 | if !ok { 48 | return nil, errors.New("missing Project ID") 49 | } 50 | 51 | // Create a new client 52 | client, err := newGoogleStorageClient(config) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | // Create a location with given config and client 58 | loc := &Location{ 59 | config: config, 60 | client: client, 61 | } 62 | 63 | return loc, nil 64 | } 65 | 66 | kindfn := func(u *url.URL) bool { 67 | return u.Scheme == Kind 68 | } 69 | 70 | stow.Register(Kind, makefn, kindfn, validatefn) 71 | } 72 | 73 | // Attempts to create a session based on the information given. 74 | func newGoogleStorageClient(config stow.Config) (*storage.Service, error) { 75 | json, _ := config.Config(ConfigJSON) 76 | var httpClient *http.Client 77 | scopes := []string{storage.DevstorageReadWriteScope} 78 | if s, ok := config.Config(ConfigScopes); ok && s != "" { 79 | scopes = strings.Split(s, ",") 80 | } 81 | if json != "" { 82 | jwtConf, err := google.JWTConfigFromJSON([]byte(json), scopes...) 83 | if err != nil { 84 | return nil, err 85 | } 86 | httpClient = jwtConf.Client(context.Background()) 87 | 88 | } else { 89 | creds, err := google.FindDefaultCredentials(context.Background(), strings.Join(scopes, ",")) 90 | if err != nil { 91 | return nil, err 92 | } 93 | httpClient = oauth2.NewClient(context.Background(), creds.TokenSource) 94 | } 95 | service, err := storage.New(httpClient) 96 | if err != nil { 97 | return nil, err 98 | } 99 | 100 | return service, nil 101 | } 102 | -------------------------------------------------------------------------------- /internal/osm/google/item.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "io" 5 | "net/url" 6 | 7 | // "strings" 8 | "time" 9 | 10 | storage "google.golang.org/api/storage/v1" 11 | ) 12 | 13 | type Item struct { 14 | container *Container // Container information is required by a few methods. 15 | client *storage.Service // A client is needed to make requests. 16 | name string 17 | hash string 18 | etag string 19 | size int64 20 | url *url.URL 21 | lastModified time.Time 22 | metadata map[string]interface{} 23 | object *storage.Object 24 | } 25 | 26 | // ID returns a string value that represents the name of a file. 27 | func (i *Item) ID() string { 28 | return i.name 29 | } 30 | 31 | // Name returns a string value that represents the name of the file. 32 | func (i *Item) Name() string { 33 | return i.name 34 | } 35 | 36 | // Size returns the size of an item in bytes. 37 | func (i *Item) Size() (int64, error) { 38 | return i.size, nil 39 | } 40 | 41 | // URL returns a url which follows the predefined format 42 | func (i *Item) URL() *url.URL { 43 | return i.url 44 | } 45 | 46 | // Open returns an io.ReadCloser to the object. Useful for downloading/streaming the object. 47 | func (i *Item) Open() (io.ReadCloser, error) { 48 | res, err := i.client.Objects.Get(i.container.name, i.name).Download() 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | return res.Body, nil 54 | } 55 | 56 | // LastMod returns the last modified date of the item. 57 | func (i *Item) LastMod() (time.Time, error) { 58 | return i.lastModified, nil 59 | } 60 | 61 | // Metadata returns a nil map and no error. 62 | // TODO: Implement this. 63 | func (i *Item) Metadata() (map[string]interface{}, error) { 64 | return i.metadata, nil 65 | } 66 | 67 | // ETag returns the ETag value 68 | func (i *Item) ETag() (string, error) { 69 | return i.etag, nil 70 | } 71 | 72 | // Object returns the Google Storage Object 73 | func (i *Item) StorageObject() *storage.Object { 74 | return i.object 75 | } 76 | 77 | // prepUrl takes a MediaLink string and returns a url 78 | func prepUrl(str string) (*url.URL, error) { 79 | u, err := url.Parse(str) 80 | if err != nil { 81 | return nil, err 82 | } 83 | u.Scheme = "google" 84 | 85 | // Discard the query string 86 | u.RawQuery = "" 87 | return u, nil 88 | } 89 | -------------------------------------------------------------------------------- /internal/osm/google/location.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "strings" 7 | 8 | "gomodules.xyz/stow" 9 | storage "google.golang.org/api/storage/v1" 10 | ) 11 | 12 | // A Location contains a client + the configurations used to create the client. 13 | type Location struct { 14 | config stow.Config 15 | client *storage.Service 16 | } 17 | 18 | func (l *Location) Service() *storage.Service { 19 | return l.client 20 | } 21 | 22 | // Close simply satisfies the Location interface. There's nothing that 23 | // needs to be done in order to satisfy the interface. 24 | func (l *Location) Close() error { 25 | return nil // nothing to close 26 | } 27 | 28 | // CreateContainer creates a new container, in this case a bucket. 29 | func (l *Location) CreateContainer(containerName string) (stow.Container, error) { 30 | 31 | projId, _ := l.config.Config(ConfigProjectId) 32 | // Create a bucket. 33 | _, err := l.client.Buckets.Insert(projId, &storage.Bucket{Name: containerName}).Do() 34 | //res, err := l.client.Buckets.Insert(projId, &storage.Bucket{Name: containerName}).Do() 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | newContainer := &Container{ 40 | name: containerName, 41 | client: l.client, 42 | } 43 | 44 | return newContainer, nil 45 | } 46 | 47 | // Containers returns a slice of the Container interface, a cursor, and an error. 48 | func (l *Location) Containers(prefix string, cursor string, count int) ([]stow.Container, string, error) { 49 | 50 | projId, _ := l.config.Config(ConfigProjectId) 51 | 52 | // List all objects in a bucket using pagination 53 | call := l.client.Buckets.List(projId).MaxResults(int64(count)) 54 | 55 | if prefix != "" { 56 | call.Prefix(prefix) 57 | } 58 | 59 | if cursor != "" { 60 | call = call.PageToken(cursor) 61 | } 62 | 63 | res, err := call.Do() 64 | if err != nil { 65 | return nil, "", err 66 | } 67 | containers := make([]stow.Container, len(res.Items)) 68 | 69 | for i, o := range res.Items { 70 | containers[i] = &Container{ 71 | name: o.Name, 72 | client: l.client, 73 | } 74 | } 75 | 76 | return containers, res.NextPageToken, nil 77 | } 78 | 79 | // Container retrieves a stow.Container based on its name which must be 80 | // exact. 81 | func (l *Location) Container(id string) (stow.Container, error) { 82 | 83 | _, err := l.client.Buckets.Get(id).Do() 84 | if err != nil { 85 | return nil, stow.ErrNotFound 86 | } 87 | 88 | c := &Container{ 89 | name: id, 90 | client: l.client, 91 | } 92 | 93 | return c, nil 94 | } 95 | 96 | // RemoveContainer removes a container simply by name. 97 | func (l *Location) RemoveContainer(id string) error { 98 | 99 | if err := l.client.Buckets.Delete(id).Do(); err != nil { 100 | return err 101 | } 102 | 103 | return nil 104 | } 105 | 106 | // ItemByURL retrieves a stow.Item by parsing the URL, in this 107 | // case an item is an object. 108 | func (l *Location) ItemByURL(url *url.URL) (stow.Item, error) { 109 | 110 | if url.Scheme != Kind { 111 | return nil, errors.New("not valid google storage URL") 112 | } 113 | 114 | // /download/storage/v1/b/stowtesttoudhratik/o/a_first%2Fthe%20item 115 | pieces := strings.SplitN(url.Path, "/", 8) 116 | 117 | c, err := l.Container(pieces[5]) 118 | if err != nil { 119 | return nil, stow.ErrNotFound 120 | } 121 | 122 | i, err := c.Item(pieces[7]) 123 | if err != nil { 124 | return nil, stow.ErrNotFound 125 | } 126 | 127 | return i, nil 128 | } 129 | -------------------------------------------------------------------------------- /internal/osm/rclone.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | core "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | ktypes "k8s.io/apimachinery/pkg/types" 10 | awsconst "kmodules.xyz/constants/aws" 11 | api "kmodules.xyz/objectstore-api/api/v1" 12 | "sigs.k8s.io/controller-runtime/pkg/client" 13 | ) 14 | 15 | // NewRcloneSecret creates a secret that contains the config file of Rclone. 16 | // So, generally, if this secret is mounted in `etc/rclone`, 17 | // the tree of `/etc/rclone` directory will be similar to, 18 | // 19 | // /etc/rclone 20 | // └── config 21 | func NewRcloneSecret(kc client.Client, name, namespace string, spec api.Backend, ownerReference []metav1.OwnerReference) (*core.Secret, error) { 22 | rcloneCtx, err := newContext(kc, spec, namespace) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | rcloneBytes := []byte(rcloneCtx) 28 | 29 | out := &core.Secret{ 30 | ObjectMeta: metav1.ObjectMeta{ 31 | Name: name, 32 | Namespace: namespace, 33 | OwnerReferences: ownerReference, 34 | }, 35 | Data: map[string][]byte{ 36 | "config": rcloneBytes, 37 | }, 38 | } 39 | return out, nil 40 | } 41 | 42 | func newContext(kc client.Client, spec api.Backend, namespace string) (string, error) { 43 | config := make(map[string][]byte) 44 | if spec.StorageSecretName != "" { 45 | secret := &core.Secret{} 46 | err := kc.Get(context.TODO(), ktypes.NamespacedName{ 47 | Name: spec.StorageSecretName, 48 | Namespace: namespace, 49 | }, secret) 50 | if err != nil { 51 | return "", err 52 | } 53 | config = secret.Data 54 | } 55 | provider, err := spec.Provider() 56 | if err != nil { 57 | return "", err 58 | } 59 | 60 | if spec.S3 != nil { 61 | return cephContext(config, provider, spec), nil 62 | } 63 | if spec.Local != nil { 64 | return localContext(provider), nil 65 | } 66 | 67 | return "", fmt.Errorf("no storage provider is configured") 68 | } 69 | 70 | func cephContext(config map[string][]byte, provider string, spec api.Backend) string { 71 | keyID := config[awsconst.AWS_ACCESS_KEY_ID] 72 | key := config[awsconst.AWS_SECRET_ACCESS_KEY] 73 | 74 | return fmt.Sprintf(`[%s] 75 | type = s3 76 | provider = Ceph 77 | env_auth = false 78 | access_key_id = %s 79 | secret_access_key = %s 80 | region = 81 | endpoint = %s 82 | location_constraint = 83 | acl = 84 | server_side_encryption = 85 | storage_class = 86 | `, provider, keyID, key, spec.S3.Endpoint) 87 | } 88 | 89 | func localContext(provider string) string { 90 | return fmt.Sprintf(`[%s] 91 | type = local 92 | `, provider) 93 | } 94 | -------------------------------------------------------------------------------- /internal/redisutil/client.go: -------------------------------------------------------------------------------- 1 | package redisutil 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/mediocregopher/radix.v2/redis" 8 | ) 9 | 10 | // IClient redis client interface 11 | type IClient interface { 12 | // Close closes the connection. 13 | Close() error 14 | 15 | // Cmd calls the given Redis command. 16 | Cmd(cmd string, args ...interface{}) *redis.Resp 17 | 18 | // PipeAppend adds the given call to the pipeline queue. 19 | // Use PipeResp() to read the response. 20 | PipeAppend(cmd string, args ...interface{}) 21 | 22 | // PipeResp returns the reply for the next request in the pipeline queue. Err 23 | // with ErrPipelineEmpty is returned if the pipeline queue is empty. 24 | PipeResp() *redis.Resp 25 | 26 | // PipeClear clears the contents of the current pipeline queue, both commands 27 | // queued by PipeAppend which have yet to be sent and responses which have yet 28 | // to be retrieved through PipeResp. The first returned int will be the number 29 | // of pending commands dropped, the second will be the number of pending 30 | // responses dropped 31 | PipeClear() (int, int) 32 | 33 | // ReadResp will read a Resp off of the connection without sending anything 34 | // first (useful after you've sent a SUSBSCRIBE command). This will block until 35 | // a reply is received or the timeout is reached (returning the IOErr). You can 36 | // use IsTimeout to check if the Resp is due to a Timeout 37 | // 38 | // Note: this is a more low-level function, you really shouldn't have to 39 | // actually use it unless you're writing your own pub/sub code 40 | ReadResp() *redis.Resp 41 | } 42 | 43 | // Client structure representing a client connection to redis 44 | type Client struct { 45 | commandsMapping map[string]string 46 | client *redis.Client 47 | } 48 | 49 | // NewClient build a client connection and connect to a redis address 50 | func NewClient(addr, password string, cnxTimeout time.Duration, commandsMapping map[string]string) (IClient, error) { 51 | var err error 52 | c := &Client{ 53 | commandsMapping: commandsMapping, 54 | } 55 | 56 | c.client, err = redis.DialTimeout("tcp", addr, cnxTimeout) 57 | if err != nil { 58 | return c, err 59 | } 60 | if password != "" { 61 | err = c.client.Cmd("AUTH", password).Err 62 | } 63 | return c, err 64 | } 65 | 66 | // Close closes the connection. 67 | func (c *Client) Close() error { 68 | return c.client.Close() 69 | } 70 | 71 | // Cmd calls the given Redis command. 72 | func (c *Client) Cmd(cmd string, args ...interface{}) *redis.Resp { 73 | return c.client.Cmd(c.getCommand(cmd), args) 74 | } 75 | 76 | // getCommand return the command name after applying rename-command 77 | func (c *Client) getCommand(cmd string) string { 78 | upperCmd := strings.ToUpper(cmd) 79 | if renamed, found := c.commandsMapping[upperCmd]; found { 80 | return renamed 81 | } 82 | return upperCmd 83 | } 84 | 85 | // PipeAppend adds the given call to the pipeline queue. 86 | func (c *Client) PipeAppend(cmd string, args ...interface{}) { 87 | c.client.PipeAppend(c.getCommand(cmd), args) 88 | } 89 | 90 | // PipeResp returns the reply for the next request in the pipeline queue. Err 91 | func (c *Client) PipeResp() *redis.Resp { 92 | return c.client.PipeResp() 93 | } 94 | 95 | // PipeClear clears the contents of the current pipeline queue 96 | func (c *Client) PipeClear() (int, int) { 97 | return c.client.PipeClear() 98 | } 99 | 100 | // ReadResp will read a Resp off of the connection without sending anything 101 | func (c *Client) ReadResp() *redis.Resp { 102 | return c.client.ReadResp() 103 | } 104 | -------------------------------------------------------------------------------- /internal/redisutil/cluster.go: -------------------------------------------------------------------------------- 1 | package redisutil 2 | 3 | import ( 4 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 5 | ) 6 | 7 | // Cluster represents a Redis Cluster 8 | type Cluster struct { 9 | Name string 10 | Namespace string 11 | Nodes map[string]*Node 12 | Status redisv1alpha1.ClusterStatus 13 | NodesPlacement redisv1alpha1.NodesPlacementInfo 14 | ActionsInfo ClusterActionsInfo 15 | } 16 | 17 | // ClusterActionsInfo use to store information about current action on the Cluster 18 | type ClusterActionsInfo struct { 19 | NbslotsToMigrate int32 20 | } 21 | 22 | // NewCluster builds and returns new Cluster instance 23 | func NewCluster(name, namespace string) *Cluster { 24 | c := &Cluster{ 25 | Name: name, 26 | Namespace: namespace, 27 | Nodes: make(map[string]*Node), 28 | } 29 | 30 | return c 31 | } 32 | 33 | // AddNode used to add new Node in the cluster 34 | // if node with the same ID is already present in the cluster 35 | // the previous Node is replaced 36 | func (c *Cluster) AddNode(node *Node) { 37 | if n, ok := c.Nodes[node.ID]; ok { 38 | n.Clear() 39 | } 40 | 41 | c.Nodes[node.ID] = node 42 | } 43 | 44 | // GetNodeByID returns a Cluster Node by its ID 45 | // if not present in the cluster return an error 46 | func (c *Cluster) GetNodeByID(id string) (*Node, error) { 47 | if n, ok := c.Nodes[id]; ok { 48 | return n, nil 49 | } 50 | return nil, nodeNotFoundedError 51 | } 52 | 53 | // GetNodeByIP returns a Cluster Node by its ID 54 | // if not present in the cluster return an error 55 | func (c *Cluster) GetNodeByIP(ip string) (*Node, error) { 56 | findFunc := func(node *Node) bool { 57 | return node.IP == ip 58 | } 59 | 60 | return c.GetNodeByFunc(findFunc) 61 | } 62 | 63 | // GetNodeByPodName returns a Cluster Node by its Pod name 64 | // if not present in the cluster return an error 65 | func (c *Cluster) GetNodeByPodName(name string) (*Node, error) { 66 | findFunc := func(node *Node) bool { 67 | if node.PodName == name { 68 | return true 69 | } 70 | return false 71 | } 72 | 73 | return c.GetNodeByFunc(findFunc) 74 | } 75 | 76 | // GetNodeByFunc returns first node found by the FindNodeFunc 77 | func (c *Cluster) GetNodeByFunc(f FindNodeFunc) (*Node, error) { 78 | for _, n := range c.Nodes { 79 | if f(n) { 80 | return n, nil 81 | } 82 | } 83 | return nil, nodeNotFoundedError 84 | } 85 | 86 | // GetNodesByFunc returns first node found by the FindNodeFunc 87 | func (c *Cluster) GetNodesByFunc(f FindNodeFunc) (Nodes, error) { 88 | nodes := Nodes{} 89 | for _, n := range c.Nodes { 90 | if f(n) { 91 | nodes = append(nodes, n) 92 | } 93 | } 94 | if len(nodes) == 0 { 95 | return nodes, nodeNotFoundedError 96 | } 97 | return nodes, nil 98 | } 99 | 100 | // FindNodeFunc function for finding a Node 101 | // it is used as input for GetNodeByFunc and GetNodesByFunc 102 | type FindNodeFunc func(node *Node) bool 103 | 104 | // ToAPIClusterStatus convert the Cluster information to a api 105 | //func (c *Cluster) ToAPIClusterStatus() redisv1alpha1.RedisClusterStatus { 106 | // status := redisv1alpha1.RedisClusterClusterStatus{} 107 | // status.Status = c.Status 108 | // for _, node := range c.Nodes { 109 | // status.Nodes = append(status.Nodes, node.ToAPINode()) 110 | // } 111 | // return status 112 | //} 113 | -------------------------------------------------------------------------------- /internal/redisutil/errors.go: -------------------------------------------------------------------------------- 1 | package redisutil 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | // Error used to represent an error 9 | type Error string 10 | 11 | func (e Error) Error() string { return string(e) } 12 | 13 | // nodeNotFoundedError returns when a node is not present in the cluster 14 | const nodeNotFoundedError = Error("node not founded") 15 | 16 | // IsNodeNotFoundedError returns true if the current error is a NodeNotFoundedError 17 | func IsNodeNotFoundedError(err error) bool { 18 | return errors.Is(err, nodeNotFoundedError) 19 | } 20 | 21 | // ClusterInfosError error type for redis cluster infos access 22 | type ClusterInfosError struct { 23 | errs map[string]error 24 | partial bool 25 | inconsistent bool 26 | } 27 | 28 | // NewClusterInfosError returns an instance of cluster infos error 29 | func NewClusterInfosError() ClusterInfosError { 30 | return ClusterInfosError{ 31 | errs: make(map[string]error), 32 | partial: false, 33 | inconsistent: false, 34 | } 35 | } 36 | 37 | // Error error string 38 | func (e ClusterInfosError) Error() string { 39 | s := "" 40 | if e.partial { 41 | s += "Cluster infos partial: " 42 | for addr, err := range e.errs { 43 | s += fmt.Sprintf("%s: '%s'", addr, err) 44 | } 45 | return s 46 | } 47 | if e.inconsistent { 48 | s += "Cluster view is inconsistent" 49 | } 50 | return s 51 | } 52 | 53 | // Partial true if some nodes of the cluster didn't answer 54 | func (e ClusterInfosError) Partial() bool { 55 | return e.partial 56 | } 57 | 58 | // Inconsistent true if the nodes do not agree with each other 59 | func (e ClusterInfosError) Inconsistent() bool { 60 | return e.inconsistent 61 | } 62 | 63 | // IsPartialError returns true if the error is due to partial data recovery 64 | func IsPartialError(err error) bool { 65 | var e ClusterInfosError 66 | ok := errors.As(err, &e) 67 | return ok && e.Partial() 68 | } 69 | 70 | // IsInconsistentError returns true if the error is due to cluster inconsistencies 71 | func IsInconsistentError(err error) bool { 72 | var e ClusterInfosError 73 | ok := errors.As(err, &e) 74 | return ok && e.Inconsistent() 75 | } 76 | -------------------------------------------------------------------------------- /internal/redisutil/node_test.go: -------------------------------------------------------------------------------- 1 | package redisutil 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestNodes_SortByFunc(t *testing.T) { 9 | n1 := Node{ 10 | ID: "n1", 11 | IP: "10.1.1.1", 12 | Port: "", 13 | Role: "master", 14 | balance: 1365, 15 | } 16 | n2 := Node{ 17 | ID: "n2", 18 | IP: "10.1.1.2", 19 | Port: "", 20 | Role: "master", 21 | balance: 1366, 22 | } 23 | n3 := Node{ 24 | ID: "n3", 25 | IP: "10.1.1.3", 26 | Port: "", 27 | Role: "master", 28 | balance: 1365, 29 | } 30 | n4 := Node{ 31 | ID: "n4", 32 | IP: "10.1.1.4", 33 | Port: "", 34 | Role: "master", 35 | balance: -4096, 36 | } 37 | type args struct { 38 | less func(*Node, *Node) bool 39 | } 40 | tests := []struct { 41 | name string 42 | n Nodes 43 | args args 44 | want Nodes 45 | }{ 46 | { 47 | name: "asc by balance", 48 | n: Nodes{&n1, &n2, &n3, &n4}, 49 | args: args{less: func(a, b *Node) bool { return a.Balance() < b.Balance() }}, 50 | want: Nodes{&n4, &n1, &n3, &n2}, 51 | }, 52 | } 53 | for _, tt := range tests { 54 | t.Run(tt.name, func(t *testing.T) { 55 | if got := tt.n.SortByFunc(tt.args.less); !reflect.DeepEqual(got, tt.want) { 56 | t.Errorf("SortByFunc() = %v, want %v", got, tt.want) 57 | } 58 | }) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /internal/redisutil/slot_test.go: -------------------------------------------------------------------------------- 1 | package redisutil 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestRemoveSlots(t *testing.T) { 9 | type args struct { 10 | slots []Slot 11 | removedSlots []Slot 12 | } 13 | tests := []struct { 14 | name string 15 | args args 16 | want []Slot 17 | }{ 18 | { 19 | name: "1", 20 | args: args{ 21 | slots: []Slot{2, 3, 4, 5, 6, 7, 8, 9, 10}, 22 | removedSlots: []Slot{2, 10}, 23 | }, 24 | want: []Slot{3, 4, 5, 6, 7, 8, 9}, 25 | }, 26 | { 27 | name: "2", 28 | args: args{ 29 | slots: []Slot{2, 5}, 30 | removedSlots: []Slot{2, 2, 3}, 31 | }, 32 | want: []Slot{5}, 33 | }, 34 | { 35 | name: "3", 36 | args: args{ 37 | slots: []Slot{0, 1, 3, 4}, 38 | removedSlots: []Slot{0, 1, 3, 4}, 39 | }, 40 | want: []Slot{}, 41 | }, 42 | { 43 | name: "4", 44 | args: args{ 45 | slots: []Slot{}, 46 | removedSlots: []Slot{2, 10}, 47 | }, 48 | want: []Slot{}, 49 | }, 50 | { 51 | name: "5", 52 | args: args{ 53 | slots: []Slot{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 54 | removedSlots: []Slot{5}, 55 | }, 56 | want: []Slot{0, 1, 2, 3, 4, 6, 7, 8, 9, 10}, 57 | }, 58 | } 59 | for _, tt := range tests { 60 | t.Run(tt.name, func(t *testing.T) { 61 | if got := RemoveSlots(tt.args.slots, tt.args.removedSlots); !reflect.DeepEqual(got, tt.want) { 62 | t.Errorf("RemoveSlots() = %v, want %v", got, tt.want) 63 | } 64 | }) 65 | } 66 | } 67 | 68 | func TestRemoveSlot(t *testing.T) { 69 | type args struct { 70 | slots []Slot 71 | removedSlot Slot 72 | } 73 | tests := []struct { 74 | name string 75 | args args 76 | want []Slot 77 | }{ 78 | { 79 | name: "1", 80 | args: args{ 81 | slots: []Slot{2, 3, 4, 5, 6, 7, 8, 9, 10}, 82 | removedSlot: 2, 83 | }, 84 | want: []Slot{3, 4, 5, 6, 7, 8, 9, 10}, 85 | }, 86 | { 87 | name: "2", 88 | args: args{ 89 | slots: []Slot{2, 5}, 90 | removedSlot: 2, 91 | }, 92 | want: []Slot{5}, 93 | }, 94 | { 95 | name: "3", 96 | args: args{ 97 | slots: []Slot{0, 1, 3, 4}, 98 | removedSlot: 3, 99 | }, 100 | want: []Slot{0, 1, 4}, 101 | }, 102 | { 103 | name: "4", 104 | args: args{ 105 | slots: []Slot{}, 106 | removedSlot: 2, 107 | }, 108 | want: []Slot{}, 109 | }, 110 | { 111 | name: "5", 112 | args: args{ 113 | slots: []Slot{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 114 | removedSlot: 5, 115 | }, 116 | want: []Slot{0, 1, 2, 3, 4, 6, 7, 8, 9, 10}, 117 | }, 118 | } 119 | for _, tt := range tests { 120 | t.Run(tt.name, func(t *testing.T) { 121 | if got := RemoveSlot(tt.args.slots, tt.args.removedSlot); !reflect.DeepEqual(got, tt.want) { 122 | t.Errorf("RemoveSlot() = %v, want %v", got, tt.want) 123 | } 124 | }) 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /internal/resources/configmaps/configmap.go: -------------------------------------------------------------------------------- 1 | package configmaps 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "sort" 7 | "strconv" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | 12 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 13 | ) 14 | 15 | const ( 16 | RestoreSucceeded = "succeeded" 17 | 18 | RedisConfKey = "redis.conf" 19 | ) 20 | 21 | // NewConfigMapForCR creates a new ConfigMap for the given Cluster 22 | func NewConfigMapForCR(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) *corev1.ConfigMap { 23 | // Do CLUSTER FAILOVER when master down 24 | shutdownContent := `#!/bin/sh 25 | CLUSTER_CONFIG="/data/nodes.conf" 26 | failover() { 27 | echo "Do CLUSTER FAILOVER" 28 | masterID=$(cat ${CLUSTER_CONFIG} | grep "myself" | awk '{print $1}') 29 | echo "Master: ${masterID}" 30 | slave=$(cat ${CLUSTER_CONFIG} | grep ${masterID} | grep "slave" | awk 'NR==1{print $2}' | sed 's/:6379@16379//') 31 | echo "Slave: ${slave}" 32 | password=$(cat /data/redis_password) 33 | if [[ -z "${password}" ]]; then 34 | redis-cli -h ${slave} CLUSTER FAILOVER 35 | else 36 | redis-cli -h ${slave} -a "${password}" CLUSTER FAILOVER 37 | fi 38 | echo "Wait for MASTER <-> SLAVE syncFinished" 39 | sleep 20 40 | } 41 | if [ -f ${CLUSTER_CONFIG} ]; then 42 | cat ${CLUSTER_CONFIG} | grep "myself" | grep "master" && \ 43 | failover 44 | fi` 45 | 46 | // Fixed Nodes.conf does not update IP address of a node when IP changes after restart, 47 | // see more https://github.com/antirez/redis/issues/4645. 48 | fixIPContent := `#!/bin/sh 49 | CLUSTER_CONFIG="/data/nodes.conf" 50 | if [ -f ${CLUSTER_CONFIG} ]; then 51 | if [ -z "${POD_IP}" ]; then 52 | echo "Unable to determine Pod IP address!" 53 | exit 1 54 | fi 55 | echo "Updating my IP to ${POD_IP} in ${CLUSTER_CONFIG}" 56 | sed -i.bak -e "/myself/ s/ .*:6379@16379/ ${POD_IP}:6379@16379/" ${CLUSTER_CONFIG} 57 | fi 58 | exec "$@"` 59 | 60 | redisConfContent := generateRedisConfContent(cluster.Spec.Config) 61 | 62 | return &corev1.ConfigMap{ 63 | ObjectMeta: metav1.ObjectMeta{ 64 | Name: RedisConfigMapName(cluster.Name), 65 | Namespace: cluster.Namespace, 66 | Labels: labels, 67 | OwnerReferences: redisv1alpha1.DefaultOwnerReferences(cluster), 68 | }, 69 | Data: map[string]string{ 70 | "shutdown.sh": shutdownContent, 71 | "fix-ip.sh": fixIPContent, 72 | RedisConfKey: redisConfContent, 73 | }, 74 | } 75 | } 76 | 77 | func generateRedisConfContent(configMap map[string]string) string { 78 | if configMap == nil { 79 | return "" 80 | } 81 | 82 | var buffer bytes.Buffer 83 | 84 | keys := make([]string, 0, len(configMap)) 85 | for k := range configMap { 86 | keys = append(keys, k) 87 | } 88 | sort.Strings(keys) 89 | 90 | for _, k := range keys { 91 | v := configMap[k] 92 | if len(v) == 0 { 93 | continue 94 | } 95 | buffer.WriteString(fmt.Sprintf("%s %s", k, v)) 96 | buffer.WriteString("\n") 97 | } 98 | 99 | return buffer.String() 100 | } 101 | 102 | func RedisConfigMapName(clusterName string) string { 103 | return fmt.Sprintf("%s-%s", "redis-cluster", clusterName) 104 | } 105 | 106 | func NewConfigMapForRestore(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) *corev1.ConfigMap { 107 | return &corev1.ConfigMap{ 108 | ObjectMeta: metav1.ObjectMeta{ 109 | Name: RestoreConfigMapName(cluster.Name), 110 | Namespace: cluster.Namespace, 111 | Labels: labels, 112 | OwnerReferences: redisv1alpha1.DefaultOwnerReferences(cluster), 113 | }, 114 | Data: map[string]string{ 115 | RestoreSucceeded: strconv.Itoa(0), 116 | }, 117 | } 118 | } 119 | 120 | func RestoreConfigMapName(clusterName string) string { 121 | return fmt.Sprintf("%s-%s", "rediscluster-restore", clusterName) 122 | } 123 | -------------------------------------------------------------------------------- /internal/resources/configmaps/configmap_test.go: -------------------------------------------------------------------------------- 1 | package configmaps 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_generateRedisConfContent(t *testing.T) { 8 | confMap := map[string]string{ 9 | "activerehashing": "yes", 10 | "appendfsync": "everysec", 11 | "appendonly": "yes", 12 | "auto-aof-rewrite-min-size": "67108864", 13 | "auto-aof-rewrite-percentage": "100", 14 | "cluster-node-timeout": "15000", 15 | "cluster-require-full-coverage": "yes", 16 | "hash-max-ziplist-entries": "512", 17 | "hash-max-ziplist-value": "64", 18 | "hll-sparse-max-bytes": "3000", 19 | "list-compress-depth": "0", 20 | "maxmemory": "1000000000", 21 | "maxmemory-policy": "noeviction", 22 | "maxmemory-samples": "5", 23 | "no-appendfsync-on-rewrite": "no", 24 | "notify-keyspace-events": "", 25 | "repl-backlog-size": "1048576", 26 | "repl-backlog-ttl": "3600", 27 | "set-max-intset-entries": "512", 28 | "slowlog-log-slower-than": "10000", 29 | "slowlog-max-len": "128", 30 | "stop-writes-on-bgsave-error": "yes", 31 | "tcp-keepalive": "0", 32 | "timeout": "0", 33 | "zset-max-ziplist-entries": "128", 34 | "zset-max-ziplist-value": "64", 35 | } 36 | want := `activerehashing yes 37 | appendfsync everysec 38 | appendonly yes 39 | auto-aof-rewrite-min-size 67108864 40 | auto-aof-rewrite-percentage 100 41 | cluster-node-timeout 15000 42 | cluster-require-full-coverage yes 43 | hash-max-ziplist-entries 512 44 | hash-max-ziplist-value 64 45 | hll-sparse-max-bytes 3000 46 | list-compress-depth 0 47 | maxmemory 1000000000 48 | maxmemory-policy noeviction 49 | maxmemory-samples 5 50 | no-appendfsync-on-rewrite no 51 | repl-backlog-size 1048576 52 | repl-backlog-ttl 3600 53 | set-max-intset-entries 512 54 | slowlog-log-slower-than 10000 55 | slowlog-max-len 128 56 | stop-writes-on-bgsave-error yes 57 | tcp-keepalive 0 58 | timeout 0 59 | zset-max-ziplist-entries 128 60 | zset-max-ziplist-value 64 61 | ` 62 | type args struct { 63 | configMap map[string]string 64 | } 65 | tests := []struct { 66 | name string 67 | args args 68 | want string 69 | }{ 70 | { 71 | name: "test", 72 | args: struct{ configMap map[string]string }{configMap: confMap}, 73 | want: want, 74 | }, 75 | } 76 | for _, tt := range tests { 77 | t.Run(tt.name, func(t *testing.T) { 78 | if got := generateRedisConfContent(tt.args.configMap); got != tt.want { 79 | t.Errorf("generateRedisConfContent()\n[%v], want\n[%v]", got, tt.want) 80 | } 81 | }) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /internal/resources/poddisruptionbudgets/poddisruptionbudget.go: -------------------------------------------------------------------------------- 1 | package poddisruptionbudgets 2 | 3 | import ( 4 | policyv11 "k8s.io/api/policy/v1" 5 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 6 | "k8s.io/apimachinery/pkg/util/intstr" 7 | 8 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 9 | ) 10 | 11 | func NewPodDisruptionBudgetForCR(cluster *redisv1alpha1.DistributedRedisCluster, name string, labels map[string]string) *policyv11.PodDisruptionBudget { 12 | maxUnavailable := intstr.FromInt(1) 13 | 14 | return &policyv11.PodDisruptionBudget{ 15 | ObjectMeta: metav1.ObjectMeta{ 16 | Labels: labels, 17 | Name: name, 18 | Namespace: cluster.Namespace, 19 | OwnerReferences: redisv1alpha1.DefaultOwnerReferences(cluster), 20 | }, 21 | Spec: policyv11.PodDisruptionBudgetSpec{ 22 | MaxUnavailable: &maxUnavailable, 23 | Selector: &metav1.LabelSelector{ 24 | MatchLabels: labels, 25 | }, 26 | }, 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /internal/resources/services/service.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | corev1 "k8s.io/api/core/v1" 5 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 6 | 7 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 8 | ) 9 | 10 | // NewHeadLessSvcForCR creates a new headless service for the given Cluster. 11 | func NewHeadLessSvcForCR(cluster *redisv1alpha1.DistributedRedisCluster, name string, labels map[string]string) *corev1.Service { 12 | clientPort := corev1.ServicePort{Name: "client", Port: 6379} 13 | gossipPort := corev1.ServicePort{Name: "gossip", Port: 16379} 14 | svc := &corev1.Service{ 15 | ObjectMeta: metav1.ObjectMeta{ 16 | Labels: labels, 17 | Name: name, 18 | Namespace: cluster.Namespace, 19 | OwnerReferences: redisv1alpha1.DefaultOwnerReferences(cluster), 20 | }, 21 | Spec: corev1.ServiceSpec{ 22 | Ports: []corev1.ServicePort{clientPort, gossipPort}, 23 | Selector: labels, 24 | ClusterIP: corev1.ClusterIPNone, 25 | }, 26 | } 27 | 28 | return svc 29 | } 30 | 31 | func NewSvcForCR(cluster *redisv1alpha1.DistributedRedisCluster, name string, labels map[string]string) *corev1.Service { 32 | var ports []corev1.ServicePort 33 | clientPort := corev1.ServicePort{Name: "client", Port: 6379} 34 | gossipPort := corev1.ServicePort{Name: "gossip", Port: 16379} 35 | if cluster.Spec.Monitor == nil { 36 | ports = append(ports, clientPort, gossipPort) 37 | } else { 38 | ports = append(ports, clientPort, gossipPort, 39 | corev1.ServicePort{Name: "prom-http", Port: cluster.Spec.Monitor.Prometheus.Port}) 40 | } 41 | 42 | svc := &corev1.Service{ 43 | ObjectMeta: metav1.ObjectMeta{ 44 | Labels: labels, 45 | Name: name, 46 | Namespace: cluster.Namespace, 47 | OwnerReferences: redisv1alpha1.DefaultOwnerReferences(cluster), 48 | }, 49 | Spec: corev1.ServiceSpec{ 50 | Ports: ports, 51 | Selector: labels, 52 | }, 53 | } 54 | 55 | return svc 56 | } 57 | -------------------------------------------------------------------------------- /internal/resources/statefulsets/helper.go: -------------------------------------------------------------------------------- 1 | package statefulsets 2 | 3 | import ( 4 | "context" 5 | 6 | appsv1 "k8s.io/api/apps/v1" 7 | corev1 "k8s.io/api/core/v1" 8 | "k8s.io/apimachinery/pkg/types" 9 | "sigs.k8s.io/controller-runtime/pkg/client" 10 | 11 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 12 | ) 13 | 14 | const passwordKey = "password" 15 | 16 | // IsPasswordChanged determine whether the password is changed. 17 | func IsPasswordChanged(cluster *redisv1alpha1.DistributedRedisCluster, sts *appsv1.StatefulSet) bool { 18 | if cluster.Spec.PasswordSecret != nil { 19 | envSet := sts.Spec.Template.Spec.Containers[0].Env 20 | secretName := getSecretKeyRefByKey(redisv1alpha1.PasswordENV, envSet) 21 | if secretName == "" { 22 | return true 23 | } 24 | if secretName != cluster.Spec.PasswordSecret.Name { 25 | return true 26 | } 27 | } 28 | return false 29 | } 30 | 31 | func getSecretKeyRefByKey(key string, envSet []corev1.EnvVar) string { 32 | for _, value := range envSet { 33 | if key == value.Name { 34 | if value.ValueFrom != nil && value.ValueFrom.SecretKeyRef != nil { 35 | return value.ValueFrom.SecretKeyRef.Name 36 | } 37 | } 38 | } 39 | return "" 40 | } 41 | 42 | // GetOldRedisClusterPassword return old redis cluster's password. 43 | func GetOldRedisClusterPassword(client client.Client, sts *appsv1.StatefulSet) (string, error) { 44 | envSet := sts.Spec.Template.Spec.Containers[0].Env 45 | secretName := getSecretKeyRefByKey(redisv1alpha1.PasswordENV, envSet) 46 | if secretName == "" { 47 | return "", nil 48 | } 49 | secret := &corev1.Secret{} 50 | err := client.Get(context.TODO(), types.NamespacedName{ 51 | Name: secretName, 52 | Namespace: sts.Namespace, 53 | }, secret) 54 | if err != nil { 55 | return "", err 56 | } 57 | return string(secret.Data[passwordKey]), nil 58 | } 59 | 60 | // GetClusterPassword return current redis cluster's password. 61 | func GetClusterPassword(client client.Client, cluster *redisv1alpha1.DistributedRedisCluster) (string, error) { 62 | if cluster.Spec.PasswordSecret == nil { 63 | return "", nil 64 | } 65 | secret := &corev1.Secret{} 66 | err := client.Get(context.TODO(), types.NamespacedName{ 67 | Name: cluster.Spec.PasswordSecret.Name, 68 | Namespace: cluster.Namespace, 69 | }, secret) 70 | if err != nil { 71 | return "", err 72 | } 73 | return string(secret.Data[passwordKey]), nil 74 | } 75 | -------------------------------------------------------------------------------- /internal/resources/statefulsets/statefulset_test.go: -------------------------------------------------------------------------------- 1 | package statefulsets 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | corev1 "k8s.io/api/core/v1" 8 | ) 9 | 10 | func Test_mergeRenameCmds(t *testing.T) { 11 | type args struct { 12 | userCmds []string 13 | systemRenameCmdMap map[string]string 14 | } 15 | tests := []struct { 16 | name string 17 | args args 18 | want []string 19 | }{ 20 | { 21 | name: "test No intersection", 22 | args: args{ 23 | userCmds: []string{ 24 | "--maxmemory 2gb", 25 | "--rename-command BGSAVE pp14qluk", 26 | "--rename-command CONFIG lni07z1p", 27 | }, 28 | systemRenameCmdMap: map[string]string{ 29 | "SAVE": "6on30p6z", 30 | "DEBUG": "8a4insyv", 31 | }, 32 | }, 33 | want: []string{ 34 | "--maxmemory 2gb", 35 | "--rename-command BGSAVE pp14qluk", 36 | "--rename-command CONFIG lni07z1p", 37 | "--rename-command DEBUG 8a4insyv", 38 | "--rename-command SAVE 6on30p6z", 39 | }, 40 | }, 41 | { 42 | name: "test intersection", 43 | args: args{ 44 | userCmds: []string{ 45 | "--rename-command BGSAVE pp14qluk", 46 | "--rename-command CONFIG lni07z1p", 47 | }, 48 | systemRenameCmdMap: map[string]string{ 49 | "BGSAVE": "fadfgad", 50 | "SAVE": "6on30p6z", 51 | "DEBUG": "8a4insyv", 52 | }, 53 | }, 54 | want: []string{ 55 | "--rename-command CONFIG lni07z1p", 56 | "--rename-command BGSAVE fadfgad", 57 | "--rename-command DEBUG 8a4insyv", 58 | "--rename-command SAVE 6on30p6z", 59 | }, 60 | }, 61 | { 62 | name: "test complex", 63 | args: args{ 64 | userCmds: []string{ 65 | "--maxmemory 2gb", 66 | "--rename-command BGSAVE pp14qluk", 67 | "--rename-command CONFIG lni07z1p", 68 | `--rename-command FLUSHALL ""`, 69 | }, 70 | systemRenameCmdMap: map[string]string{ 71 | "BGSAVE": "fadfgad", 72 | "SAVE": "6on30p6z", 73 | "DEBUG": "8a4insyv", 74 | }, 75 | }, 76 | want: []string{ 77 | "--maxmemory 2gb", 78 | "--rename-command CONFIG lni07z1p", 79 | `--rename-command FLUSHALL ""`, 80 | "--rename-command BGSAVE fadfgad", 81 | "--rename-command DEBUG 8a4insyv", 82 | "--rename-command SAVE 6on30p6z", 83 | }, 84 | }, 85 | } 86 | for _, tt := range tests { 87 | t.Run(tt.name, func(t *testing.T) { 88 | if got := mergeRenameCmds(tt.args.userCmds, tt.args.systemRenameCmdMap); !reflect.DeepEqual(got, tt.want) { 89 | t.Errorf("mergeRenameCmds() = %v, want %v", got, tt.want) 90 | } 91 | }) 92 | } 93 | } 94 | 95 | func Test_customContainerEnv(t *testing.T) { 96 | type args struct { 97 | env []corev1.EnvVar 98 | customEnv []corev1.EnvVar 99 | } 100 | tests := []struct { 101 | name string 102 | args args 103 | want []corev1.EnvVar 104 | }{ 105 | { 106 | name: "nil all", 107 | args: args{ 108 | env: nil, 109 | customEnv: nil, 110 | }, 111 | want: nil, 112 | }, 113 | { 114 | name: "nil env", 115 | args: args{ 116 | env: nil, 117 | customEnv: []corev1.EnvVar{{ 118 | Name: "foo", 119 | Value: "", 120 | ValueFrom: nil, 121 | }}, 122 | }, 123 | want: []corev1.EnvVar{{ 124 | Name: "foo", 125 | Value: "", 126 | ValueFrom: nil, 127 | }}, 128 | }, 129 | { 130 | name: "nil custom env", 131 | args: args{ 132 | customEnv: nil, 133 | env: []corev1.EnvVar{{ 134 | Name: "foo", 135 | Value: "", 136 | ValueFrom: nil, 137 | }}, 138 | }, 139 | want: []corev1.EnvVar{{ 140 | Name: "foo", 141 | Value: "", 142 | ValueFrom: nil, 143 | }}, 144 | }, 145 | { 146 | name: "env for bar", 147 | args: args{ 148 | env: []corev1.EnvVar{{ 149 | Name: "foo", 150 | Value: "", 151 | ValueFrom: nil, 152 | }}, 153 | customEnv: []corev1.EnvVar{{ 154 | Name: "bar", 155 | Value: "", 156 | ValueFrom: nil, 157 | }}, 158 | }, 159 | want: []corev1.EnvVar{{ 160 | Name: "foo", 161 | Value: "", 162 | ValueFrom: nil, 163 | }, { 164 | Name: "bar", 165 | Value: "", 166 | ValueFrom: nil, 167 | }}, 168 | }, 169 | } 170 | for _, tt := range tests { 171 | t.Run(tt.name, func(t *testing.T) { 172 | if got := customContainerEnv(tt.args.env, tt.args.customEnv); !reflect.DeepEqual(got, tt.want) { 173 | t.Errorf("customContainerEnv() = %v, want %v", got, tt.want) 174 | } 175 | }) 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /internal/utils/compare.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/go-logr/logr" 7 | ) 8 | 9 | func CompareIntValue(name string, old, new *int32, reqLogger logr.Logger) bool { 10 | if old == nil && new == nil { 11 | return true 12 | } else if old == nil || new == nil { 13 | return false 14 | } else if *old != *new { 15 | reqLogger.V(4).Info(fmt.Sprintf("compare status.%s: %d - %d", name, *old, *new)) 16 | return true 17 | } 18 | 19 | return false 20 | } 21 | 22 | func CompareInt32(name string, old, new int32, reqLogger logr.Logger) bool { 23 | if old != new { 24 | reqLogger.V(4).Info(fmt.Sprintf("compare status.%s: %d - %d", name, old, new)) 25 | return true 26 | } 27 | 28 | return false 29 | } 30 | 31 | func CompareStringValue(name string, old, new string, reqLogger logr.Logger) bool { 32 | if old != new { 33 | reqLogger.V(4).Info(fmt.Sprintf("compare %s: %s - %s", name, old, new)) 34 | return true 35 | } 36 | 37 | return false 38 | } 39 | -------------------------------------------------------------------------------- /internal/utils/labels.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | // MergeLabels merges all the label maps received as argument into a single new label map. 4 | func MergeLabels(allLabels ...map[string]string) map[string]string { 5 | res := map[string]string{} 6 | 7 | for _, labels := range allLabels { 8 | if labels != nil { 9 | for k, v := range labels { 10 | res[k] = v 11 | } 12 | } 13 | } 14 | return res 15 | } 16 | -------------------------------------------------------------------------------- /internal/utils/math.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "math" 4 | 5 | func Round(num float64) int { 6 | return int(num + math.Copysign(0.5, num)) 7 | } 8 | -------------------------------------------------------------------------------- /internal/utils/parse.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | ) 7 | 8 | func ParseRedisMemConf(p string) (string, error) { 9 | var mul int64 = 1 10 | u := strings.ToLower(p) 11 | digits := u 12 | 13 | if strings.HasSuffix(u, "k") { 14 | digits = u[:len(u)-len("k")] 15 | mul = 1000 16 | } else if strings.HasSuffix(u, "kb") { 17 | digits = u[:len(u)-len("kb")] 18 | mul = 1024 19 | } else if strings.HasSuffix(u, "m") { 20 | digits = u[:len(u)-len("m")] 21 | mul = 1000 * 1000 22 | } else if strings.HasSuffix(u, "mb") { 23 | digits = u[:len(u)-len("mb")] 24 | mul = 1024 * 1024 25 | } else if strings.HasSuffix(u, "g") { 26 | digits = u[:len(u)-len("g")] 27 | mul = 1000 * 1000 * 1000 28 | } else if strings.HasSuffix(u, "gb") { 29 | digits = u[:len(u)-len("gb")] 30 | mul = 1024 * 1024 * 1024 31 | } else if strings.HasSuffix(u, "b") { 32 | digits = u[:len(u)-len("b")] 33 | mul = 1 34 | } 35 | 36 | val, err := strconv.ParseInt(digits, 10, 64) 37 | if err != nil { 38 | return "", err 39 | } 40 | 41 | return strconv.FormatInt(val*mul, 10), nil 42 | } 43 | -------------------------------------------------------------------------------- /internal/utils/parse_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "testing" 4 | 5 | func TestParseRedisMemConf(t *testing.T) { 6 | type args struct { 7 | p string 8 | } 9 | tests := []struct { 10 | name string 11 | args args 12 | want string 13 | wantErr bool 14 | }{ 15 | { 16 | name: "b", 17 | args: args{ 18 | p: "12b", 19 | }, 20 | want: "12", 21 | wantErr: false, 22 | }, 23 | { 24 | name: "digit", 25 | args: args{ 26 | p: "1202", 27 | }, 28 | want: "1202", 29 | wantErr: false, 30 | }, 31 | { 32 | name: "B", 33 | args: args{ 34 | p: "12B", 35 | }, 36 | want: "12", 37 | wantErr: false, 38 | }, 39 | { 40 | name: "k", 41 | args: args{ 42 | p: "12k", 43 | }, 44 | want: "12000", 45 | wantErr: false, 46 | }, 47 | { 48 | name: "kk", 49 | args: args{ 50 | p: "12kk", 51 | }, 52 | want: "", 53 | wantErr: true, 54 | }, 55 | { 56 | name: "kb", 57 | args: args{ 58 | p: "12kb", 59 | }, 60 | want: "12288", 61 | wantErr: false, 62 | }, 63 | { 64 | name: "Kb", 65 | args: args{ 66 | p: "12Kb", 67 | }, 68 | want: "12288", 69 | wantErr: false, 70 | }, 71 | { 72 | name: "m", 73 | args: args{ 74 | p: "12m", 75 | }, 76 | want: "12000000", 77 | wantErr: false, 78 | }, 79 | { 80 | name: "mB", 81 | args: args{ 82 | p: "12mb", 83 | }, 84 | want: "12582912", 85 | wantErr: false, 86 | }, 87 | { 88 | name: "g", 89 | args: args{ 90 | p: "12g", 91 | }, 92 | want: "12000000000", 93 | wantErr: false, 94 | }, 95 | { 96 | name: "gb", 97 | args: args{ 98 | p: "12gb", 99 | }, 100 | want: "12884901888", 101 | wantErr: false, 102 | }, 103 | } 104 | for _, tt := range tests { 105 | t.Run(tt.name, func(t *testing.T) { 106 | got, err := ParseRedisMemConf(tt.args.p) 107 | if (err != nil) != tt.wantErr { 108 | t.Errorf("ParseRedisMemConf() error = %v, wantErr %v", err, tt.wantErr) 109 | return 110 | } 111 | if got != tt.want { 112 | t.Errorf("ParseRedisMemConf() got = %v, want %v", got, tt.want) 113 | } 114 | }) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /internal/utils/rename_cmd.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | "github.com/go-logr/logr" 10 | ) 11 | 12 | // BuildCommandReplaceMapping reads the config file with the command-replace lines and build a mapping of 13 | // bad lines are ignored silently 14 | func BuildCommandReplaceMapping(filePath string, log logr.Logger) map[string]string { 15 | mapping := make(map[string]string) 16 | log.Info("Building Command Replace Mapping", "FilePath", filePath) 17 | if filePath == "" { 18 | return mapping 19 | } 20 | log.Info("Building Command Replace Mapping", "FilePath Not EMPTY", filePath) 21 | file, err := os.Open(filePath) 22 | if err != nil { 23 | log.Error(err, fmt.Sprintf("cannot open %s", filePath)) 24 | return mapping 25 | } 26 | defer file.Close() 27 | 28 | scanner := bufio.NewScanner(file) 29 | for scanner.Scan() { 30 | elems := strings.Fields(scanner.Text()) 31 | if len(elems) == 3 && strings.ToLower(elems[0]) == "rename-command" { 32 | mapping[strings.ToUpper(elems[1])] = elems[2] 33 | } 34 | } 35 | 36 | if err := scanner.Err(); err != nil { 37 | log.Error(err, fmt.Sprintf("cannot parse %s", filePath)) 38 | return mapping 39 | } 40 | return mapping 41 | } 42 | -------------------------------------------------------------------------------- /internal/utils/scoped.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | const ( 4 | // AnnotationScope annotation name for defining instance scope. Used for specifying cluster wide clusters. 5 | // A namespace-scoped operator watches and manages resources in a single namespace, whereas a cluster-scoped operator watches and manages resources cluster-wide. 6 | AnnotationScope = "redis.kun/scope" 7 | //AnnotationClusterScoped annotation value for cluster wide clusters. 8 | AnnotationClusterScoped = "cluster-scoped" 9 | ) 10 | 11 | var isClusterScoped = true 12 | 13 | func IsClusterScoped() bool { 14 | return isClusterScoped 15 | } 16 | 17 | func SetClusterScoped(namespace string) { 18 | if namespace != "" { 19 | isClusterScoped = false 20 | } 21 | } 22 | 23 | func ShoudManage(annotations map[string]string) bool { 24 | if v, ok := annotations[AnnotationScope]; ok { 25 | if IsClusterScoped() { 26 | return v == AnnotationClusterScoped 27 | } 28 | } else { 29 | if !IsClusterScoped() { 30 | return true 31 | } 32 | } 33 | return false 34 | } 35 | -------------------------------------------------------------------------------- /internal/utils/string.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | // Stringer implement the string interface 4 | type Stringer interface { 5 | String() string 6 | } 7 | 8 | // SliceJoin concatenates the elements of a to create a single string. The separator string 9 | // sep is placed between elements in the resulting string. 10 | func SliceJoin(a []Stringer, sep string) string { 11 | switch len(a) { 12 | case 0: 13 | return "" 14 | case 1: 15 | return a[0].String() 16 | case 2: 17 | // Special case for common small values. 18 | // Remove if golang.org/issue/6714 is fixed 19 | return a[0].String() + sep + a[1].String() 20 | case 3: 21 | // Special case for common small values. 22 | // Remove if golang.org/issue/6714 is fixed 23 | return a[0].String() + sep + a[1].String() + sep + a[2].String() 24 | } 25 | n := len(sep) * (len(a) - 1) 26 | for i := 0; i < len(a); i++ { 27 | n += len(a[i].String()) 28 | } 29 | 30 | b := make([]byte, n) 31 | bp := copy(b, a[0].String()) 32 | for _, s := range a[1:] { 33 | bp += copy(b[bp:], sep) 34 | bp += copy(b[bp:], s.String()) 35 | } 36 | return string(b) 37 | } 38 | -------------------------------------------------------------------------------- /internal/utils/types.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | // Int32 returns the value of the int32 pointer passed in or 4 | // 0 if the pointer is nil. 5 | func Int32(v *int32) int32 { 6 | if v != nil { 7 | return *v 8 | } 9 | return 0 10 | } 11 | -------------------------------------------------------------------------------- /static/redis-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TykTechnologies/redis-cluster-operator/c5df83bad6e71a41bdf62c0a58187bd9b5d00e76/static/redis-cluster.png -------------------------------------------------------------------------------- /test/e2e/README.md: -------------------------------------------------------------------------------- 1 | # E2E Tests for Redis Cluster Operator 2 | 3 | This directory contains end-to-end tests for the `redis-cluster-operator` using Ginkgo and the Operator SDK. 4 | 5 | ## What It Does 6 | 7 | #### DRC operator test 8 | - Builds and loads the operator image into a Kind cluster 9 | - Installs CRDs 10 | - Deploys the operator 11 | - Verifies the controller-manager pod is running 12 | - Cleans up the test namespace after execution 13 | #### DRC CRUD test 14 | Below is a concise documentation of what the tests verify: 15 | 16 | - Creates a DistributedRedisCluster resource with a random name and password; waits for the cluster to stabilize and seeds test data. 17 | - Changes the Redis configuration, updates the resource, and confirms consistent data state. 18 | - Simulates master pod deletion and verifies automatic recovery and data integrity. 19 | - Tests scaling operations by scaling up then scaling down, ensuring the cluster remains healthy. 20 | - Resets the cluster password and performs a minor version rolling update, verifying stability and consistent data throughout. 21 | 22 | ## Prerequisites 23 | 24 | - Docker 25 | - Kind (cluster named `test`) 26 | - Kubectl 27 | - Go 28 | - [Ginkgo CLI (optional)](https://onsi.github.io/ginkgo/#ginkgo-cli) 29 | 30 | ## Run the Tests 31 | 32 | Below is a command-based short documentation on how to run the tests locally: 33 | 34 | --- 35 | 36 | ### How to Run the E2E Tests 37 | 38 | 39 | 1. **Set Up the Environment** 40 | - **Create a Kind Cluster:** 41 | ```bash 42 | kind create cluster --name e2e-test --image kindest/node:v1.31.6 43 | kubectl cluster-info && kubectl get nodes 44 | ``` 45 | 46 | 2. **Run Operator E2E Tests** 47 | ```bash 48 | go test ./test/e2e/drc_operator -v -ginkgo.v 49 | ``` 50 | 51 | 3. **Deploy the Operator & Prepare CRUD Test** 52 | ```bash 53 | # Load operator image into Kind 54 | # The 'tykio/redis-cluster-operator:v0.0.0-teste2e' image is built as part of the Operator E2E Tests, so it does not need to be built again. 55 | kind load docker-image tykio/redis-cluster-operator:v0.0.0-teste2e --name e2e-test 56 | 57 | # Install CRDs and deploy the operator 58 | make install 59 | make deploy IMG=tykio/redis-cluster-operator:v0.0.0-teste2e 60 | 61 | # Build and load the CRUD test image 62 | make docker-build-e2e IMG=tykio/drc-crud-test:v0.0.0-teste2e 63 | kind load docker-image tykio/drc-crud-test:v0.0.0-teste2e --name e2e-test 64 | 65 | # Wait for the operator to be available 66 | kubectl wait --for=condition=available --timeout=90s deployment/redis-cluster-operator-controller-manager --namespace redis-cluster-operator-system 67 | ``` 68 | 69 | 4. **Run and Monitor the CRUD E2E Test Job** 70 | ```bash 71 | # Apply the job definition to run CRUD tests 72 | kubectl apply -f test/e2e/drc_crud/job.yaml 73 | 74 | # Optional: Get the pod name 75 | POD=$(kubectl get pods --namespace redis-cluster-operator-system -l job-name=drc-crud-e2e-job -o jsonpath="{.items[0].metadata.name}") 76 | 77 | # Stream logs from the test pod 78 | kubectl logs --namespace redis-cluster-operator-system -f "$POD" 79 | 80 | # Check job status (succeeded or failed) 81 | kubectl get job drc-crud-e2e-job --namespace redis-cluster-operator-system 82 | ``` -------------------------------------------------------------------------------- /test/e2e/drc_crud/drc/client.go: -------------------------------------------------------------------------------- 1 | package drc 2 | 3 | import ( 4 | appsv1 "k8s.io/api/apps/v1" 5 | corev1 "k8s.io/api/core/v1" 6 | rbacv1 "k8s.io/api/rbac/v1" 7 | "k8s.io/apimachinery/pkg/runtime" 8 | "k8s.io/client-go/rest" 9 | "sigs.k8s.io/controller-runtime/pkg/client" 10 | 11 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 12 | ) 13 | 14 | func NewClient(config *rest.Config) (client.Client, error) { 15 | // Create a new scheme rather than using the global scheme. 16 | sch := runtime.NewScheme() 17 | _ = corev1.AddToScheme(sch) 18 | _ = rbacv1.AddToScheme(sch) 19 | _ = appsv1.AddToScheme(sch) 20 | 21 | // Add your API types to the scheme and check for errors. 22 | if err := redisv1alpha1.AddToScheme(sch); err != nil { 23 | return nil, err 24 | } 25 | 26 | // Set up client options with your custom scheme and mapper. 27 | options := client.Options{ 28 | Scheme: sch, 29 | } 30 | 31 | // Create and return a new client instance. 32 | cli, err := client.New(config, options) 33 | if err != nil { 34 | return nil, err 35 | } 36 | return cli, nil 37 | } 38 | -------------------------------------------------------------------------------- /test/e2e/drc_crud/drc/goredis_util.go: -------------------------------------------------------------------------------- 1 | package drc 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/go-redis/redis/v8" 9 | uuid "github.com/satori/go.uuid" 10 | "golang.org/x/sync/errgroup" 11 | ) 12 | 13 | const defaultTimeOut = time.Second * 30 14 | 15 | // GoRedis contains ClusterClient. 16 | type GoRedis struct { 17 | client *redis.ClusterClient 18 | password string 19 | } 20 | 21 | // NewGoRedis return a new ClusterClient. 22 | func NewGoRedis(addr, password string) *GoRedis { 23 | return &GoRedis{ 24 | client: redis.NewClusterClient(&redis.ClusterOptions{ 25 | Addrs: []string{addr}, 26 | Password: password, 27 | //MaxRetries: 5, 28 | // 29 | //PoolSize: 3, 30 | //MinIdleConns: 1, 31 | //PoolTimeout: defaultTimeOut, 32 | //IdleTimeout: defaultTimeOut, 33 | }), 34 | password: password, 35 | } 36 | } 37 | 38 | // StuffingData filled with (round * n)'s key. 39 | func (g *GoRedis) StuffingData(round, n int) error { 40 | ctx, cancel := context.WithTimeout(context.Background(), defaultTimeOut) 41 | defer cancel() 42 | 43 | var group errgroup.Group 44 | for i := 0; i < round; i++ { 45 | // Capture the current value of i for the goroutine. 46 | i := i 47 | group.Go(func() error { 48 | // Expired timestamp: 1 hour ago. 49 | expiredTime := time.Now().Unix() - 3600 50 | // Future timestamp: 1 hour from now. 51 | futureTime := time.Now().Unix() + 3600 52 | 53 | for j := 0; j < n; j++ { 54 | // Generate key with a round number and a new UUID. 55 | key := fmt.Sprintf("apikey-%s-%d", uuid.NewV4().String(), i) 56 | var value string 57 | // 5 and 8 are random numbers. if n is 2000: 58 | // 400 keys with a future expiration timestamp. 59 | // 200 keys with a dummy session and an expired timestamp. 60 | // 1400 keys with just an expired timestamp. 61 | if j%5 == 0 { 62 | value = fmt.Sprintf("{\"expires\": %d}", futureTime) 63 | } else if j%8 == 0 && j%5 != 0 { 64 | // Use a dummy session and an expired timestamp. 65 | value = fmt.Sprintf("{\"TykJWTSessionID\": \"dummy-session\", \"expires\": %d}", expiredTime) 66 | } else { 67 | value = fmt.Sprintf("{\"expires\": %d}", expiredTime) 68 | } 69 | 70 | if err := g.client.Set(ctx, key, value, 0).Err(); err != nil { 71 | return err 72 | } 73 | } 74 | return nil 75 | }) 76 | } 77 | return group.Wait() 78 | } 79 | 80 | // PrintClusterState prints the Redis cluster nodes and info after scaling. 81 | func (g *GoRedis) PrintClusterState(scaledCount int) error { 82 | ctx, cancel := context.WithTimeout(context.Background(), defaultTimeOut) 83 | defer cancel() 84 | 85 | // Fetch CLUSTER NODES 86 | nodes, err := g.client.ClusterNodes(ctx).Result() 87 | if err != nil { 88 | return fmt.Errorf("failed to fetch cluster nodes: %w", err) 89 | } 90 | fmt.Printf("\n=== After scaling to %d nodes: CLUSTER NODES ===\n%s\n", scaledCount, nodes) 91 | 92 | // Fetch CLUSTER INFO 93 | info, err := g.client.ClusterInfo(ctx).Result() 94 | if err != nil { 95 | return fmt.Errorf("failed to fetch cluster info: %w", err) 96 | } 97 | fmt.Printf("\n=== After scaling to %d nodes: CLUSTER INFO ===\n%s\n", scaledCount, info) 98 | 99 | return nil 100 | } 101 | 102 | // DBSize return DBsize of all master nodes. 103 | func (g *GoRedis) DBSize() (int64, error) { 104 | ctx, cancel := context.WithTimeout(context.Background(), defaultTimeOut) 105 | defer cancel() 106 | return g.client.DBSize(ctx).Result() 107 | } 108 | 109 | // Password return redis password. 110 | func (g *GoRedis) Password() string { 111 | return g.password 112 | } 113 | 114 | // Close closes the cluster client. 115 | func (g *GoRedis) Close() error { 116 | return g.client.Close() 117 | } 118 | -------------------------------------------------------------------------------- /test/e2e/drc_crud/drc/rename.conf: -------------------------------------------------------------------------------- 1 | rename-command CONFIG lni07z1p -------------------------------------------------------------------------------- /test/e2e/drc_crud/e2e_suite_test.go: -------------------------------------------------------------------------------- 1 | package drc_crud_test 2 | 3 | import ( 4 | "testing" 5 | 6 | ctrl "sigs.k8s.io/controller-runtime" 7 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 8 | 9 | . "github.com/onsi/ginkgo/v2" 10 | . "github.com/onsi/gomega" 11 | 12 | redisv1alpha1 "github.com/TykTechnologies/redis-cluster-operator/api/v1alpha1" 13 | drctest "github.com/TykTechnologies/redis-cluster-operator/test/e2e/drc_crud/drc" 14 | ) 15 | 16 | var f *drctest.Framework 17 | var drc *redisv1alpha1.DistributedRedisCluster 18 | 19 | func TestDrc(t *testing.T) { 20 | RegisterFailHandler(Fail) 21 | RunSpecs(t, "Drc Suite") 22 | } 23 | 24 | var _ = BeforeSuite(func() { 25 | ctrl.SetLogger(zap.New(zap.UseDevMode(true))) 26 | f = drctest.NewFramework("test") 27 | if err := f.BeforeEach(); err != nil { 28 | f.Failf("Framework BeforeEach err: %s", err.Error()) 29 | } 30 | }) 31 | 32 | var _ = AfterSuite(func() { 33 | if err := f.DeleteRedisCluster(drc); err != nil { 34 | f.Logf("deleting DistributedRedisCluster err: %s", err.Error()) 35 | } 36 | if err := f.AfterEach(); err != nil { 37 | f.Failf("Framework AfterSuite err: %s", err.Error()) 38 | } 39 | }) 40 | -------------------------------------------------------------------------------- /test/e2e/drc_crud/job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | # Unique name for the ClusterRole 5 | name: drc-crud-job-access 6 | rules: 7 | - apiGroups: [""] 8 | resources: ["namespaces"] 9 | verbs: ["get", "list", "create", "update", "patch", "delete"] 10 | --- 11 | apiVersion: rbac.authorization.k8s.io/v1 12 | kind: ClusterRoleBinding 13 | metadata: 14 | name: drc-crud-job-binding 15 | subjects: 16 | - kind: ServiceAccount 17 | name: redis-cluster-operator-controller-manager 18 | namespace: redis-cluster-operator-system 19 | roleRef: 20 | kind: ClusterRole 21 | name: drc-crud-job-access 22 | apiGroup: rbac.authorization.k8s.io 23 | --- 24 | apiVersion: batch/v1 25 | kind: Job 26 | metadata: 27 | name: drc-crud-e2e-job 28 | namespace: redis-cluster-operator-system 29 | spec: 30 | backoffLimit: 4 31 | template: 32 | metadata: 33 | name: drc-drud-e2e-pod 34 | spec: 35 | serviceAccountName: redis-cluster-operator-controller-manager 36 | containers: 37 | - name: drc-crud 38 | image: tykio/drc-crud-test:v0.0.0-teste2e 39 | imagePullPolicy: IfNotPresent 40 | volumeMounts: 41 | - name: redisconf 42 | mountPath: /etc/redisconf 43 | args: 44 | - --rename-command-path=/etc/redisconf 45 | - --rename-command-file=redis.conf 46 | volumes: 47 | - name: redisconf 48 | configMap: 49 | name: redis-cluster-operator-redisconf 50 | restartPolicy: Never 51 | -------------------------------------------------------------------------------- /test/e2e/drc_operator/e2e_suite_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2025. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | ... 6 | */ 7 | 8 | package drc_operator_test 9 | 10 | import ( 11 | "fmt" 12 | "os" 13 | "os/exec" 14 | "testing" 15 | "time" 16 | 17 | . "github.com/onsi/ginkgo/v2" 18 | . "github.com/onsi/gomega" 19 | ctrl "sigs.k8s.io/controller-runtime" 20 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 21 | 22 | "github.com/TykTechnologies/redis-cluster-operator/test/utils" 23 | ) 24 | 25 | var ( 26 | namespace = "redis-cluster-operator-system" 27 | projectimage = "tykio/redis-cluster-operator:v0.0.0-teste2e" 28 | ) 29 | 30 | func TestE2E(t *testing.T) { 31 | RegisterFailHandler(Fail) 32 | fmt.Fprintln(GinkgoWriter, "Starting redis-cluster-operator suite") 33 | RunSpecs(t, "Operator and DRC E2E Suite") 34 | } 35 | 36 | var _ = BeforeSuite(func() { 37 | 38 | ctrl.SetLogger(zap.New(zap.UseDevMode(true))) 39 | 40 | if v, ok := os.LookupEnv("IMG"); ok { 41 | projectimage = v 42 | } 43 | 44 | By("creating manager namespace") 45 | cmd := exec.Command("kubectl", "create", "namespace", namespace) 46 | _, err := utils.Run(cmd) 47 | Expect(err).NotTo(HaveOccurred()) 48 | 49 | By("building the manager (Operator) image") 50 | cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) 51 | _, err = utils.Run(cmd) 52 | Expect(err).NotTo(HaveOccurred()) 53 | 54 | By("loading the manager (Operator) image on Kind") 55 | err = utils.LoadImageToKindClusterWithName(projectimage) 56 | Expect(err).NotTo(HaveOccurred()) 57 | 58 | By("installing CRDs") 59 | cmd = exec.Command("make", "install") 60 | _, err = utils.Run(cmd) 61 | Expect(err).NotTo(HaveOccurred()) 62 | 63 | By("deploying the controller-manager") 64 | cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) 65 | _, err = utils.Run(cmd) 66 | Expect(err).NotTo(HaveOccurred()) 67 | 68 | // Optional sleep to let the operator settle 69 | time.Sleep(5 * time.Second) 70 | }) 71 | 72 | var _ = AfterSuite(func() { 73 | 74 | By("undeploy the controller-manager") 75 | cmd := exec.Command("make", "undeploy") 76 | _, err := utils.Run(cmd) 77 | Expect(err).NotTo(HaveOccurred()) 78 | 79 | }) 80 | -------------------------------------------------------------------------------- /test/e2e/drc_operator/e2e_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2025. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | ... 6 | */ 7 | 8 | package drc_operator_test 9 | 10 | import ( 11 | "fmt" 12 | "os/exec" 13 | "time" 14 | 15 | . "github.com/onsi/ginkgo/v2" 16 | . "github.com/onsi/gomega" 17 | 18 | "github.com/TykTechnologies/redis-cluster-operator/test/utils" 19 | ) 20 | 21 | var _ = Describe("Operator Controller", func() { 22 | It("should have the controller-manager pod running", func() { 23 | var controllerPodName string 24 | verifyControllerUp := func() error { 25 | // Get pod name 26 | 27 | cmd := exec.Command("kubectl", "get", 28 | "pods", "-l", "control-plane=controller-manager", 29 | "-o", "go-template={{ range .items }}"+ 30 | "{{ if not .metadata.deletionTimestamp }}"+ 31 | "{{ .metadata.name }}"+ 32 | "{{ \"\\n\" }}{{ end }}{{ end }}", 33 | "-n", namespace, 34 | ) 35 | 36 | podOutput, err := utils.Run(cmd) 37 | ExpectWithOffset(2, err).NotTo(HaveOccurred()) 38 | podNames := utils.GetNonEmptyLines(string(podOutput)) 39 | if len(podNames) != 1 { 40 | return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) 41 | } 42 | controllerPodName = podNames[0] 43 | ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) 44 | 45 | // Validate pod status 46 | cmd = exec.Command("kubectl", "get", 47 | "pods", controllerPodName, "-o", "jsonpath={.status.phase}", 48 | "-n", namespace, 49 | ) 50 | status, err := utils.Run(cmd) 51 | ExpectWithOffset(2, err).NotTo(HaveOccurred()) 52 | if string(status) != "Running" { 53 | return fmt.Errorf("controller pod in %s status", status) 54 | } 55 | return nil 56 | } 57 | Eventually(verifyControllerUp, time.Minute, time.Second).Should(Succeed()) 58 | }) 59 | }) 60 | -------------------------------------------------------------------------------- /test/e2e/drcb/drcb_suite_test.go: -------------------------------------------------------------------------------- 1 | package drcb_test 2 | 3 | // import ( 4 | // "testing" 5 | 6 | // . "github.com/onsi/ginkgo" 7 | // . "github.com/onsi/gomega" 8 | 9 | // redisv1alpha1 "github.com/ucloud/redis-cluster-operator/pkg/apis/redis/v1alpha1" 10 | // "github.com/ucloud/redis-cluster-operator/test/e2e" 11 | // ) 12 | 13 | // var f *e2e.Framework 14 | // var drc *redisv1alpha1.DistributedRedisCluster 15 | // var rdrc *redisv1alpha1.DistributedRedisCluster 16 | // var drcb *redisv1alpha1.RedisClusterBackup 17 | 18 | // func TestDrcb(t *testing.T) { 19 | // RegisterFailHandler(Fail) 20 | // RunSpecs(t, "Drcb Suite") 21 | // } 22 | 23 | // var _ = BeforeSuite(func() { 24 | // f = e2e.NewFramework("drcb") 25 | // if err := f.BeforeEach(); err != nil { 26 | // f.Failf("Framework BeforeEach err: %s", err.Error()) 27 | // } 28 | // }) 29 | 30 | // var _ = AfterSuite(func() { 31 | // if err := f.DeleteRedisCluster(rdrc); err != nil { 32 | // f.Logf("deleting DistributedRedisCluster err: %s", err.Error()) 33 | // } 34 | // if err := f.AfterEach(); err != nil { 35 | // f.Failf("Framework AfterSuite err: %s", err.Error()) 36 | // } 37 | // }) 38 | --------------------------------------------------------------------------------