├── .github ├── dependabot.yml └── workflows │ ├── build-test.yaml │ ├── cla.yaml │ ├── lint.yaml │ └── release.yaml ├── .gitignore ├── .golangci.yaml ├── .markdownlint.yaml ├── .yamllint ├── CODE-OF-CONDUCT.md ├── CODEOWNERS ├── CONTRIBUTING.md ├── DCO ├── Dockerfile ├── Dockerfile.release ├── LICENSE ├── NOTICE ├── README.md ├── cmd └── spicedb-operator │ └── main.go ├── config ├── crds │ ├── authzed.com_spicedbclusters.yaml │ └── kustomization.yaml ├── kustomization.yaml ├── operator.yaml ├── rbac │ ├── kustomization.yaml │ ├── role.yaml │ ├── spicedb-operator-edit.yaml │ └── spicedb-operator-view.yaml └── update-graph.yaml ├── e2e ├── assertions_test.go ├── cluster_test.go ├── databases │ ├── cockroach.go │ ├── database.go │ ├── manifests │ │ ├── cockroachdb.yaml │ │ ├── mysql.yaml │ │ ├── postgres.yaml │ │ └── spanner.yaml │ ├── mysql.go │ ├── postgres.go │ └── spanner.go ├── e2e_test.go ├── go.mod ├── go.sum ├── matchers_test.go ├── util │ └── port_forward.go └── util_test.go ├── examples ├── .gitignore └── cockroachdb-tls-ingress │ ├── README.md │ ├── cert-manager │ ├── cert-manager.yaml │ └── kustomization.yaml │ ├── contour │ ├── contour.yaml │ └── kustomization.yaml │ ├── database │ ├── crdb.yaml │ ├── kustomization.yaml │ └── namespace.yaml │ ├── ingress │ ├── issuer.yaml │ └── kustomization.yaml │ ├── kustomization.yaml │ └── spicedb │ ├── ingress.yaml │ ├── issuer.yaml │ ├── kustomization.yaml │ └── spicedb.yaml ├── go.mod ├── go.sum ├── goreleaser.yaml ├── magefiles └── magefile.go ├── pkg ├── apis │ └── authzed │ │ ├── register.go │ │ └── v1alpha1 │ │ ├── conditions.go │ │ ├── doc.go │ │ ├── register.go │ │ ├── status.go │ │ ├── types.go │ │ └── zz_generated.deepcopy.go ├── cmd │ └── run │ │ └── run.go ├── config │ ├── config.go │ ├── config_test.go │ ├── global.go │ ├── keys.go │ ├── keys_test.go │ ├── patch.go │ ├── patch_test.go │ └── testdata │ │ ├── swagger.1.26.3.json │ │ └── swagger.1.30.2.json ├── controller │ ├── check_migrations.go │ ├── check_migrations_test.go │ ├── cleanup_job.go │ ├── cleanup_job_test.go │ ├── client.go │ ├── config_change.go │ ├── context.go │ ├── controller.go │ ├── controller_test.go │ ├── ensure_deployment.go │ ├── ensure_deployment_test.go │ ├── pause.go │ ├── pause_test.go │ ├── run_migration.go │ ├── run_migration_test.go │ ├── secret_adoption.go │ ├── secret_adoption_test.go │ ├── self_pause.go │ ├── validate_config.go │ ├── validate_config_test.go │ ├── wait_for_migrations.go │ └── wait_for_migrations_test.go ├── crds │ ├── authzed.com_spicedbclusters.yaml │ └── crds.go ├── metadata │ ├── keys.go │ └── pause.go ├── updates │ ├── file.go │ ├── file_test.go │ ├── memory.go │ ├── memory_test.go │ └── source.go └── version │ └── version.go ├── proposed-update-graph.yaml └── tools ├── generate-update-graph ├── main.go └── main_test.go ├── go.mod └── go.sum /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | updates: 4 | - package-ecosystem: "gomod" 5 | directory: "/" 6 | schedule: 7 | interval: "monthly" 8 | labels: 9 | - "area/dependencies" 10 | groups: 11 | gomod-version: 12 | patterns: ["*"] 13 | - package-ecosystem: "docker" 14 | directory: "/" 15 | schedule: 16 | interval: "monthly" 17 | labels: 18 | - "area/dependencies" 19 | groups: 20 | docker-version: 21 | patterns: ["*"] 22 | - package-ecosystem: "github-actions" 23 | directory: "/" 24 | schedule: 25 | interval: "monthly" 26 | groups: 27 | actions-version: 28 | patterns: ["*"] 29 | -------------------------------------------------------------------------------- /.github/workflows/build-test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Build & Test" 3 | on: # yamllint disable-line rule:truthy 4 | push: 5 | branches: 6 | - "main" 7 | merge_group: 8 | types: 9 | - "checks_requested" 10 | pull_request: 11 | branches: 12 | - "*" 13 | env: 14 | GO_VERSION: "~1.22" 15 | jobs: 16 | paths-filter: 17 | runs-on: "ubuntu-latest" 18 | outputs: 19 | codechange: "${{ steps.filter.outputs.codechange }}" 20 | graphchange: "${{ steps.graph-filter.outputs.graphchange }}" 21 | steps: 22 | - uses: "actions/checkout@v4" 23 | - uses: "dorny/paths-filter@v3" 24 | id: "filter" 25 | with: 26 | filters: | 27 | codechange: 28 | - ".github/workflows/build-test.yaml" 29 | - "Dockerfile" 30 | - "go.mod" 31 | - "go.sum" 32 | - "cmd/**" 33 | - "pkg/**" 34 | - "e2e/**" 35 | - "internal/**" 36 | - uses: "dorny/paths-filter@v3" 37 | id: "graph-filter" 38 | with: 39 | filters: | 40 | graphchange: 41 | - "proposed-update-graph.yaml" 42 | build: 43 | needs: "paths-filter" 44 | if: | 45 | needs.paths-filter.outputs.codechange == 'true' 46 | name: "Build Binary" 47 | runs-on: "ubuntu-latest" 48 | steps: 49 | - uses: "actions/checkout@v4" 50 | - uses: "authzed/actions/setup-go@main" 51 | with: 52 | go-version: "${{ env.GO_VERSION }}" 53 | - uses: "authzed/actions/go-build@main" 54 | 55 | image-build: 56 | needs: "paths-filter" 57 | if: | 58 | needs.paths-filter.outputs.codechange == 'true' 59 | name: "Build Container Image" 60 | runs-on: "ubuntu-latest" 61 | steps: 62 | - uses: "actions/checkout@v4" 63 | - uses: "authzed/actions/setup-go@main" 64 | with: 65 | go-version: "${{ env.GO_VERSION }}" 66 | - uses: "authzed/actions/docker-build@main" 67 | with: 68 | push: false 69 | tags: "authzed/spicedb-operator:ci" 70 | buildx: false 71 | qemu: false 72 | 73 | unit: 74 | needs: "paths-filter" 75 | if: | 76 | needs.paths-filter.outputs.codechange == 'true' 77 | name: "Unit Tests" 78 | runs-on: "ubuntu-latest" 79 | steps: 80 | - uses: "actions/checkout@v4" 81 | with: 82 | submodules: true 83 | - uses: "authzed/actions/setup-go@main" 84 | with: 85 | go-version: "${{ env.GO_VERSION }}" 86 | - uses: "docker/setup-qemu-action@v3" 87 | - uses: "docker/setup-buildx-action@v3" 88 | - name: "Run Unit Tests" 89 | uses: "magefile/mage-action@v3" 90 | with: 91 | version: "latest" 92 | args: "test:unit" 93 | 94 | e2e: 95 | needs: "paths-filter" 96 | if: | 97 | needs.paths-filter.outputs.codechange == 'true' || needs.paths-filter.outputs.graphchange == 'true' 98 | name: "E2E Tests" 99 | runs-on: "ubuntu-latest-8-cores" 100 | steps: 101 | - uses: "actions/checkout@v4" 102 | if: | 103 | needs.paths-filter.outputs.graphchange == 'true' 104 | with: 105 | submodules: true 106 | token: "${{ secrets.AUTHZED_BOT_PAT }}" 107 | repository: "${{ github.event.pull_request.head.repo.full_name }}" 108 | ref: "${{ github.event.pull_request.head.ref }}" 109 | - uses: "actions/checkout@v4" 110 | if: | 111 | needs.paths-filter.outputs.graphchange == 'false' 112 | with: 113 | submodules: true 114 | - uses: "authzed/actions/setup-go@main" 115 | with: 116 | go-version: "${{ env.GO_VERSION }}" 117 | - uses: "docker/setup-qemu-action@v3" 118 | - uses: "docker/setup-buildx-action@v3" 119 | - name: "Run E2E Tests" 120 | uses: "magefile/mage-action@v3" 121 | with: 122 | version: "latest" 123 | args: "test:e2e" 124 | - name: "Check if validated update graph has changed" 125 | if: | 126 | needs.paths-filter.outputs.graphchange == 'true' 127 | uses: "tj-actions/verify-changed-files@v20" 128 | id: "verify-changed-graph" 129 | with: 130 | files: | 131 | config/update-graph.yaml 132 | - name: "Commit validated update graph" 133 | uses: "EndBug/add-and-commit@v9" 134 | if: | 135 | steps.verify-changed-graph.outputs.files_changed == 'true' 136 | with: 137 | committer_name: "GitHub Actions" 138 | committer_email: "41898282+github-actions[bot]@users.noreply.github.com" 139 | default_author: "github_actor" 140 | message: "update validated graph after successful tests" 141 | pathspec_error_handling: "exitImmediately" 142 | - uses: "actions/upload-artifact@v4" 143 | if: "always()" 144 | # this upload step is really flaky, don't fail the job if it fails 145 | continue-on-error: true 146 | with: 147 | name: "cluster-state" 148 | path: "e2e/cluster-state" 149 | -------------------------------------------------------------------------------- /.github/workflows/cla.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "CLA" 3 | on: # yamllint disable-line rule:truthy 4 | issue_comment: 5 | types: 6 | - "created" 7 | pull_request_target: 8 | types: 9 | - "opened" 10 | - "closed" 11 | - "synchronize" 12 | merge_group: 13 | types: 14 | - "checks_requested" 15 | jobs: 16 | cla: 17 | name: "Check Signature" 18 | runs-on: "ubuntu-latest" 19 | steps: 20 | - uses: "authzed/actions/cla-check@main" 21 | with: 22 | github_token: "${{ secrets.GITHUB_TOKEN }}" 23 | cla_assistant_token: "${{ secrets.CLA_ASSISTANT_ACCESS_TOKEN }}" 24 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Lint" 3 | on: # yamllint disable-line rule:truthy 4 | push: 5 | branches: 6 | - "!dependabot/*" 7 | - "main" 8 | pull_request: 9 | branches: ["*"] 10 | merge_group: 11 | types: 12 | - "checks_requested" 13 | env: 14 | GO_VERSION: "~1.22" 15 | jobs: 16 | go-lint: 17 | name: "Lint Go" 18 | runs-on: "ubuntu-latest" 19 | steps: 20 | - uses: "actions/checkout@v4" 21 | - uses: "authzed/actions/setup-go@main" 22 | with: 23 | go-version: "${{ env.GO_VERSION }}" 24 | - name: "Go Format" 25 | # using this instead of the authzed/actions version because `.` 26 | # properly ignores counterfeiter codegen 27 | working-directory: "magefiles" 28 | run: "go run mvdan.cc/gofumpt -w ." 29 | - name: "Codegen" 30 | uses: "magefile/mage-action@v3" 31 | with: 32 | version: "latest" 33 | args: "generate" 34 | - name: "Verify Gofumpt" 35 | uses: "chainguard-dev/actions/nodiff@main" 36 | with: 37 | fixup-command: "gofumpt" 38 | - uses: "authzed/actions/go-mod-tidy@main" 39 | - uses: "authzed/actions/go-mod-tidy@main" 40 | with: 41 | working_directory: "./tools" 42 | - uses: "authzed/actions/go-mod-tidy@main" 43 | with: 44 | working_directory: "./magefiles" 45 | - uses: "authzed/actions/go-mod-tidy@main" 46 | with: 47 | working_directory: "./e2e" 48 | - uses: "authzed/actions/go-generate@main" 49 | - uses: "authzed/actions/golangci-lint@main" 50 | 51 | extra-lint: 52 | name: "Lint YAML & Markdown" 53 | runs-on: "ubuntu-latest" 54 | steps: 55 | - uses: "actions/checkout@v4" 56 | - uses: "authzed/actions/yaml-lint@main" 57 | - uses: "stefanprodan/kube-tools@v1" 58 | with: 59 | command: "kustomize build ./config" 60 | # Disabled due to issues with Kustomize, see: 61 | # - https://github.com/instrumenta/kubeval-action/pull/3 62 | # - https://github.com/instrumenta/kubeval/issues/232 63 | # - uses: "instrumenta/kubeval-action@5915e4adba5adccac07cb156b82e54c3fed74921" 64 | # with: 65 | # files: "config" 66 | - uses: "authzed/actions/markdown-lint@main" 67 | 68 | codeql: 69 | if: "${{ github.event_name == 'pull_request' }}" 70 | name: "Analyze with CodeQL" 71 | runs-on: "ubuntu-latest-8-cores" 72 | permissions: 73 | actions: "read" 74 | contents: "read" 75 | security-events: "write" 76 | strategy: 77 | fail-fast: false 78 | matrix: 79 | language: ["go"] 80 | steps: 81 | - uses: "actions/checkout@v4" 82 | - uses: "authzed/actions/codeql@main" 83 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Release" 3 | on: # yamllint disable-line rule:truthy 4 | push: 5 | tags: 6 | - "*" 7 | permissions: 8 | contents: "write" 9 | packages: "write" 10 | env: 11 | GO_VERSION: "~1.22" 12 | jobs: 13 | goreleaser: 14 | runs-on: "ubuntu-latest" 15 | env: 16 | KUSTOMIZER_ARTIFACT: "oci://ghcr.io/${{github.repository_owner}}/${{github.event.repository.name}}-manifests" 17 | steps: 18 | - uses: "actions/checkout@v4" 19 | with: 20 | fetch-depth: 0 21 | - uses: "authzed/actions/setup-go@main" 22 | with: 23 | go-version: "${{ env.GO_VERSION }}" 24 | - uses: "authzed/actions/docker-login@main" 25 | with: 26 | quayio_token: "${{ secrets.QUAYIO_PASSWORD }}" 27 | github_token: "${{ secrets.GITHUB_TOKEN }}" 28 | dockerhub_token: "${{ secrets.DOCKERHUB_ACCESS_TOKEN }}" 29 | - uses: "docker/setup-qemu-action@v3" 30 | - uses: "docker/setup-buildx-action@v3" 31 | # the release directory is gitignored, which keeps goreleaser from 32 | # complaining about a dirty tree 33 | - name: "Copy manifests to release directory" 34 | run: | 35 | mkdir release 36 | cp -R config release 37 | - name: "Set operator image in release manifests" 38 | uses: "mikefarah/yq@master" 39 | with: 40 | cmd: | 41 | yq eval '.images[0].newTag="${{ github.ref_name }}"' -i ./release/config/kustomization.yaml 42 | - name: "Build release bundle.yaml" 43 | uses: "karancode/kustomize-github-action@master" 44 | with: 45 | token: "${{ github.token }}" 46 | kustomize_build_dir: "release/config" 47 | kustomize_output_file: "release/bundle.yaml" 48 | - uses: "goreleaser/goreleaser-action@v6" 49 | with: 50 | distribution: "goreleaser-pro" 51 | version: "latest" 52 | args: "release --clean" 53 | env: 54 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 55 | GORELEASER_KEY: "${{ secrets.GORELEASER_KEY }}" 56 | - name: "Setup Kustomizer CLI" 57 | uses: "stefanprodan/kustomizer/action@main" 58 | - name: "Push release manifests" 59 | run: | 60 | kustomizer push artifact ${KUSTOMIZER_ARTIFACT}:${{ github.ref_name }} -k ./release/config \ 61 | --source=${{ github.repositoryUrl }} \ 62 | --revision="${{ github.ref_name }}/${{ github.sha }}" 63 | - name: "Tag latest release manifests" 64 | run: | 65 | kustomizer tag artifact ${KUSTOMIZER_ARTIFACT}:${GITHUB_REF_NAME} latest 66 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | testbin/ 3 | release/ 4 | *.kubeconfig 5 | *.test 6 | e2e/cluster-state/** 7 | e2e/*.lck 8 | e2e/*.lck.* 9 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | run: 3 | timeout: "5m" 4 | output: 5 | sort-results: true 6 | linters-settings: 7 | goimports: 8 | local-prefixes: "github.com/authzed/spicedb-operator" 9 | stylecheck: 10 | dot-import-whitelist: 11 | - "github.com/onsi/ginkgo/v2" 12 | - "github.com/onsi/gomega" 13 | linters: 14 | enable: 15 | - "bidichk" 16 | - "bodyclose" 17 | - "errcheck" 18 | - "errname" 19 | - "errorlint" 20 | - "gofumpt" 21 | - "goimports" 22 | - "goprintffuncname" 23 | - "gosec" 24 | - "gosimple" 25 | - "govet" 26 | - "importas" 27 | - "ineffassign" 28 | - "makezero" 29 | - "prealloc" 30 | - "predeclared" 31 | - "promlinter" 32 | - "revive" 33 | - "rowserrcheck" 34 | - "staticcheck" 35 | - "stylecheck" 36 | - "tenv" 37 | - "typecheck" 38 | - "unconvert" 39 | - "unused" 40 | - "wastedassign" 41 | - "whitespace" 42 | -------------------------------------------------------------------------------- /.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | line-length: false 3 | no-hard-tabs: false 4 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | # vim: ft=yaml 2 | --- 3 | ignore: | 4 | config/ 5 | proposed-update-graph.yaml 6 | e2e/ 7 | examples 8 | pkg/crds 9 | yaml-files: 10 | - "*.yaml" 11 | - "*.yml" 12 | - ".yamllint" 13 | extends: "default" 14 | rules: 15 | quoted-strings: "enable" 16 | line-length: "disable" 17 | -------------------------------------------------------------------------------- /CODE-OF-CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities. 4 | 5 | We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or nationality. 6 | 7 | Examples of unacceptable behavior by participants include: 8 | 9 | - The use of sexualized language or imagery 10 | - Personal attacks 11 | - Trolling or insulting/derogatory comments 12 | - Public or private harassment 13 | - Publishing other’s private information, such as physical or electronic addresses, without explicit permission 14 | - Other unethical or unprofessional conduct 15 | 16 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. 17 | By adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently applying these principles to every aspect of managing this project. 18 | Project maintainers who do not follow or enforce the Code of Conduct may be permanently removed from the project team. 19 | 20 | This code of conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. 21 | 22 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers. 23 | 24 | This Code of Conduct is adapted from the Contributor Covenant, version 1.2.0, available [here](https://www.contributor-covenant.org/version/1/2/0/code-of-conduct.html) 25 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @authzed/spicedb-maintainers 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | ## Communication 4 | 5 | - Bug Reports & Feature Requests: [GitHub Issues] 6 | - Questions: [GitHub Discussions] or [Discord] 7 | 8 | All communication in these forums abides by our [Code of Conduct]. 9 | 10 | [GitHub Issues]: https://github.com/authzed/spicedb-operator/issues 11 | [Code of Conduct]: CODE-OF-CONDUCT.md 12 | [Github Discussions]: https://github.com/orgs/authzed/discussions/new?category=q-a 13 | [Discord]: https://authzed.com/discord 14 | 15 | ## Creating issues 16 | 17 | If any part of the project has a bug or documentation mistakes, please let us know by opening an issue. 18 | All bugs and mistakes are considered very seriously, regardless of complexity. 19 | 20 | Before creating an issue, please check that an issue reporting the same problem does not already exist. 21 | To make the issue accurate and easy to understand, please try to create issues that are: 22 | 23 | - Unique -- do not duplicate existing bug report. 24 | Duplicate bug reports will be closed. 25 | - Specific -- include as much details as possible: which version, what environment, what configuration, etc. 26 | - Reproducible -- include the steps to reproduce the problem. 27 | Some issues might be hard to reproduce, so please do your best to include the steps that might lead to the problem. 28 | - Isolated -- try to isolate and reproduce the bug with minimum dependencies. 29 | It would significantly slow down the speed to fix a bug if too many dependencies are involved in a bug report. 30 | Debugging external systems that rely on this project is out of scope, but guidance or help using the project itself is fine. 31 | - Scoped -- one bug per report. 32 | Do not follow up with another bug inside one report. 33 | 34 | It may be worthwhile to read [Elika Etemad’s article on filing good bug reports][filing-good-bugs] before creating a bug report. 35 | 36 | Maintainers might ask for further information to resolve an issue. 37 | 38 | [filing-good-bugs]: http://fantasai.inkedblade.net/style/talks/filing-good-bugs/ 39 | 40 | ## Finding issues 41 | 42 | You can find issues by priority: [Urgent], [High], [Medium], [Low], [Maybe]. 43 | There are also [good first issues]. 44 | 45 | [Urgent]: https://github.com/authzed/spicedb/labels/priority%2F0%20urgent 46 | [High]: https://github.com/authzed/spicedb/labels/priority%2F1%20high 47 | [Medium]: https://github.com/authzed/spicedb/labels/priority%2F2%20medium 48 | [Low]: https://github.com/authzed/spicedb/labels/priority%2F3%20low 49 | [Maybe]: https://github.com/authzed/spicedb/labels/priority%2F4%20maybe 50 | [good first issues]: https://github.com/authzed/spicedb/labels/hint%2Fgood%20first%20issue 51 | 52 | ## Contribution flow 53 | 54 | This is a rough outline of what a contributor's workflow looks like: 55 | 56 | - Create an issue 57 | - Fork the project 58 | - Create a [feature branch] 59 | - Push changes to your branch 60 | - Submit a pull request 61 | - Respond to feedback from project maintainers 62 | - Rebase to squash related and fixup commits 63 | - Get LGTM from reviewer(s) 64 | - Merge with a merge commit 65 | 66 | Creating new issues is one of the best ways to contribute. 67 | You have no obligation to offer a solution or code to fix an issue that you open. 68 | If you do decide to try and contribute something, please submit an issue first so that a discussion can occur to avoid any wasted efforts. 69 | 70 | [feature branch]: https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow 71 | 72 | ## Legal requirements 73 | 74 | In order to protect the project, all contributors are required to sign our [Contributor License Agreement][cla] before their contribution is accepted. 75 | 76 | The signing process has been automated by [CLA Assistant][cla-assistant] during the Pull Request review process and only requires responding with a comment acknowledging the agreement. 77 | 78 | [cla]: https://github.com/authzed/cla/blob/main/v1/icla.md 79 | [cla-assistant]: https://github.com/cla-assistant/cla-assistant 80 | 81 | ## Common tasks 82 | 83 | ### Testing & building a binary 84 | 85 | In order to build and test the project, the [latest stable version of Go] and knowledge of a [working Go environment] are required. 86 | 87 | [latest stable version of Go]: https://golang.org/dl 88 | [working Go environment]: https://golang.org/doc/code.html 89 | 90 | Install [mage](https://magefile.org/#installation): 91 | 92 | ```sh 93 | # homebrew, see link for other options 94 | brew install mage 95 | ``` 96 | 97 | Run e2e tests: 98 | 99 | ```sh 100 | mage test:e2e 101 | ``` 102 | 103 | Run unit tests: 104 | 105 | ```sh 106 | mage test:unit 107 | ``` 108 | 109 | ### Adding dependencies 110 | 111 | This project does not use anything other than the standard [Go modules] toolchain for managing dependencies. 112 | 113 | [Go modules]: https://golang.org/ref/mod 114 | 115 | ```sh 116 | go get github.com/org/newdependency@version 117 | ``` 118 | 119 | Continuous integration enforces that `go mod tidy` has been run. 120 | 121 | ### Regenerating `proposed-update-graph.yaml` 122 | 123 | The update graph can be regenerated whenever there is a new spicedb release. 124 | CI will validate all new edges when there are changes to `proposed-update-graph.yaml` and will copy them into `config/update-graph.yaml` if successful. 125 | 126 | ```go 127 | mage gen:graph 128 | ``` 129 | -------------------------------------------------------------------------------- /DCO: -------------------------------------------------------------------------------- 1 | Developer Certificate of Origin 2 | Version 1.1 3 | 4 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 5 | 1 Letterman Drive 6 | Suite D4700 7 | San Francisco, CA, 94129 8 | 9 | Everyone is permitted to copy and distribute verbatim copies of this 10 | license document, but changing it is not allowed. 11 | 12 | 13 | Developer's Certificate of Origin 1.1 14 | 15 | By making a contribution to this project, I certify that: 16 | 17 | (a) The contribution was created in whole or in part by me and I 18 | have the right to submit it under the open source license 19 | indicated in the file; or 20 | 21 | (b) The contribution is based upon previous work that, to the best 22 | of my knowledge, is covered under an appropriate open source 23 | license and I have the right under that license to submit that 24 | work with modifications, whether created in whole or in part 25 | by me, under the same open source license (unless I am 26 | permitted to submit under a different license), as indicated 27 | in the file; or 28 | 29 | (c) The contribution was provided directly to me by some other 30 | person who certified (a), (b) or (c) and I have not modified 31 | it. 32 | 33 | (d) I understand and agree that this project and the contribution 34 | are public and that a record of the contribution (including all 35 | personal information I submit with it, including my sign-off) is 36 | maintained indefinitely and may be redistributed consistent with 37 | this project or the open source license(s) involved. 38 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24-alpine AS builder 2 | WORKDIR /go/src/app 3 | ENV CGO_ENABLED=0 4 | 5 | COPY go.mod go.sum ./ 6 | COPY . . 7 | RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod go build ./cmd/... 8 | 9 | FROM cgr.dev/chainguard/static:latest 10 | 11 | COPY --from=builder /go/src/app/spicedb-operator /usr/local/bin/spicedb-operator 12 | ENTRYPOINT ["spicedb-operator"] 13 | -------------------------------------------------------------------------------- /Dockerfile.release: -------------------------------------------------------------------------------- 1 | # vim: syntax=dockerfile 2 | FROM cgr.dev/chainguard/static:latest 3 | COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.25 /ko-app/grpc-health-probe /usr/local/bin/grpc_health_probe 4 | COPY spicedb-operator /usr/local/bin/spicedb-operator 5 | ENTRYPOINT ["spicedb-operator"] 6 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | SpiceDB Operator 2 | Copyright 2022 Authzed, Inc 3 | 4 | This product includes software developed at 5 | Authzed, Inc. (https://www.authzed.com/). 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SpiceDB Operator 2 | 3 | [![Container Image](https://img.shields.io/github/v/release/authzed/spicedb-operator?color=%232496ED&label=container&logo=docker "Container Image")](https://hub.docker.com/r/authzed/spicedb-operator/tags) 4 | [![Docs](https://img.shields.io/badge/docs-authzed.com-%234B4B6C "Authzed Documentation")](https://docs.authzed.com) 5 | [![Build Status](https://github.com/authzed/spicedb-operator/workflows/Build%20&%20Test/badge.svg "GitHub Actions")](https://github.com/authzed/spicedb-operator/actions) 6 | [![Discord Server](https://img.shields.io/discord/844600078504951838?color=7289da&logo=discord "Discord Server")](https://discord.gg/jTysUaxXzM) 7 | [![Twitter](https://img.shields.io/twitter/follow/authzed?color=%23179CF0&logo=twitter&style=flat-square "@authzed on Twitter")](https://twitter.com/authzed) 8 | 9 | A [Kubernetes operator] for managing [SpiceDB] clusters. 10 | 11 | Features include: 12 | 13 | - Creation, management, and scaling of SpiceDB clusters with a single [Custom Resource] 14 | - Automated datastore migrations when upgrading SpiceDB versions 15 | 16 | Have questions? Join our [Discord]. 17 | 18 | Looking to contribute? See [CONTRIBUTING.md]. 19 | 20 | [Kubernetes operator]: https://kubernetes.io/docs/concepts/extend-kubernetes/operator/ 21 | [SpiceDB]: https://github.com/authzed/spicedb 22 | [Custom Resource]: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ 23 | [Discord]: https://authzed.com/discord 24 | [CONTRIBUTING.md]: CONTRIBUTING.md 25 | 26 | ## Getting Started 27 | 28 | In order to get started, you'll need a Kubernetes cluster. 29 | For local development, install your tool of choice. 30 | You can use whatever, so long as you're comfortable with it and it works on your platform. 31 | We recommend one of the following: 32 | 33 | - [Docker Desktop](https://www.docker.com/products/docker-desktop/) 34 | - [kind](https://kind.sigs.k8s.io) 35 | - [minikube](https://minikube.sigs.k8s.io) 36 | 37 | Next, you'll install a [release](https://github.com/authzed/spicedb-operator/releases/) of the operator: 38 | 39 | ```console 40 | kubectl apply --server-side -f https://github.com/authzed/spicedb-operator/releases/latest/download/bundle.yaml 41 | ``` 42 | 43 | Finally you can create your first cluster: 44 | 45 | ```console 46 | kubectl apply --server-side -f - < 0 { 144 | staticSpiceDBController, err := static.NewStaticController[*v1alpha1.SpiceDBCluster]( 145 | logger, 146 | "static-spicedbs", 147 | o.BootstrapSpicedbsPath, 148 | v1alpha1ClusterGVR, 149 | dclient) 150 | if err != nil { 151 | return err 152 | } 153 | controllers = append(controllers, staticSpiceDBController) 154 | } 155 | 156 | ctrl, err := controller.NewController(ctx, registry, dclient, kclient, resources, o.OperatorConfigPath, broadcaster, o.WatchNamespaces) 157 | if err != nil { 158 | return err 159 | } 160 | controllers = append(controllers, ctrl) 161 | 162 | // register with metrics collector 163 | spiceDBClusterMetrics := ctrlmetrics.NewConditionStatusCollector[*v1alpha1.SpiceDBCluster](o.MetricNamespace, "clusters", v1alpha1.SpiceDBClusterResourceName) 164 | 165 | if len(o.WatchNamespaces) == 0 { 166 | o.WatchNamespaces = []string{corev1.NamespaceAll} 167 | } 168 | for _, n := range o.WatchNamespaces { 169 | lister := typed.MustListerForKey[*v1alpha1.SpiceDBCluster](registry, typed.NewRegistryKey(controller.OwnedFactoryKey(n), v1alpha1ClusterGVR)) 170 | spiceDBClusterMetrics.AddListerBuilder(func() ([]*v1alpha1.SpiceDBCluster, error) { 171 | return lister.List(labels.Everything()) 172 | }) 173 | } 174 | legacyregistry.CustomMustRegister(spiceDBClusterMetrics) 175 | 176 | if ctx.Err() != nil { 177 | return ctx.Err() 178 | } 179 | 180 | mgr := manager.NewManager(o.DebugFlags.DebuggingConfiguration, o.DebugAddress, broadcaster, eventSink) 181 | 182 | return mgr.Start(ctx, make(chan struct{}, 1), controllers...) 183 | } 184 | 185 | // DisableClientRateLimits removes rate limiting against the apiserver; we 186 | // respect priority and fairness and will back off if the server tells us to 187 | func DisableClientRateLimits(restConfig *rest.Config) { 188 | restConfig.Burst = 2000 189 | restConfig.QPS = -1 190 | } 191 | -------------------------------------------------------------------------------- /pkg/config/global.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "github.com/authzed/spicedb-operator/pkg/updates" 4 | 5 | // OperatorConfig holds operator-wide config that is used across all objects 6 | type OperatorConfig struct { 7 | ImageName string `json:"imageName,omitempty"` 8 | updates.UpdateGraph 9 | } 10 | 11 | func NewOperatorConfig() OperatorConfig { 12 | return OperatorConfig{ 13 | UpdateGraph: updates.UpdateGraph{ 14 | Channels: make([]updates.Channel, 0), 15 | }, 16 | } 17 | } 18 | 19 | func (o OperatorConfig) Copy() OperatorConfig { 20 | return OperatorConfig{ 21 | ImageName: o.ImageName, 22 | UpdateGraph: o.UpdateGraph.Copy(), 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /pkg/config/keys.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | func newKey[V comparable](k string, defaultValue V) *key[V] { 10 | return &key[V]{ 11 | key: k, 12 | defaultValue: defaultValue, 13 | } 14 | } 15 | 16 | func newStringKey(key string) *key[string] { 17 | return newKey(key, "") 18 | } 19 | 20 | func (k *key[V]) peek(config RawConfig) any { 21 | v, ok := config[k.key] 22 | if !ok { 23 | return k.defaultValue 24 | } 25 | return v 26 | } 27 | 28 | func (k *key[V]) pop(config RawConfig) V { 29 | v := k.peek(config) 30 | delete(config, k.key) 31 | tv, ok := v.(V) 32 | if !ok { 33 | return k.defaultValue 34 | } 35 | 36 | return tv 37 | } 38 | 39 | type intOrStringKey[I int64 | int32 | int16 | int8] struct { 40 | key string 41 | defaultValue I 42 | } 43 | 44 | func newIntOrStringKey[I int64 | int32 | int16 | int8](key string, defaultValue I) *intOrStringKey[I] { 45 | return &intOrStringKey[I]{ 46 | key: key, 47 | defaultValue: defaultValue, 48 | } 49 | } 50 | 51 | func (k *intOrStringKey[I]) pop(config RawConfig) (out I, err error) { 52 | v, ok := config[k.key] 53 | delete(config, k.key) 54 | if !ok { 55 | return k.defaultValue, nil 56 | } 57 | 58 | var bits int 59 | switch any(out).(type) { 60 | case int8: 61 | bits = 8 62 | case int16: 63 | bits = 16 64 | case int32: 65 | bits = 32 66 | case int64: 67 | bits = 64 68 | default: 69 | panic("invalid int type") 70 | } 71 | 72 | var parsed int64 73 | parsed, err = strconv.ParseInt(fmt.Sprint(v), 10, bits) 74 | if err != nil { 75 | return 76 | } 77 | out = I(parsed) 78 | return 79 | } 80 | 81 | type boolOrStringKey struct { 82 | key string 83 | defaultValue bool 84 | } 85 | 86 | func newBoolOrStringKey(key string, defaultValue bool) *boolOrStringKey { 87 | return &boolOrStringKey{ 88 | key: key, 89 | defaultValue: defaultValue, 90 | } 91 | } 92 | 93 | func (k *boolOrStringKey) pop(config RawConfig) (out bool, err error) { 94 | v, ok := config[k.key] 95 | delete(config, k.key) 96 | if !ok { 97 | return k.defaultValue, nil 98 | } 99 | 100 | switch value := v.(type) { 101 | case string: 102 | out, err = strconv.ParseBool(value) 103 | if err != nil { 104 | return 105 | } 106 | case bool: 107 | out = value 108 | default: 109 | err = fmt.Errorf("expected bool or string for key %s", k.key) 110 | } 111 | return 112 | } 113 | 114 | type metadataSetKey string 115 | 116 | func (k metadataSetKey) pop(config RawConfig, objectType, metadataType string) (metadata map[string]string, warnings []error, err error) { 117 | v, ok := config[string(k)] 118 | delete(config, string(k)) 119 | if !ok { 120 | return 121 | } 122 | 123 | metadata = make(map[string]string) 124 | 125 | switch value := v.(type) { 126 | case string: 127 | if len(value) > 0 { 128 | extraMetadataPairs := strings.Split(value, ",") 129 | for _, p := range extraMetadataPairs { 130 | k, v, ok := strings.Cut(p, "=") 131 | if !ok { 132 | warnings = append(warnings, fmt.Errorf("couldn't parse extra %s %s %q: values should be of the form k=v,k2=v2", objectType, metadataType, p)) 133 | continue 134 | } 135 | metadata[k] = v 136 | } 137 | } 138 | case map[string]any: 139 | for k, v := range value { 140 | metadataValue, ok := v.(string) 141 | if !ok { 142 | warnings = append(warnings, fmt.Errorf("couldn't parse extra %s %s %v", objectType, metadataType, v)) 143 | continue 144 | } 145 | metadata[k] = metadataValue 146 | } 147 | default: 148 | err = fmt.Errorf("expected string or map for key %s", k) 149 | } 150 | return 151 | } 152 | -------------------------------------------------------------------------------- /pkg/config/keys_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | var emptyConfig = RawConfig{} 10 | 11 | func TestStringKey(t *testing.T) { 12 | for _, val := range []struct { 13 | description string 14 | value any 15 | expected any 16 | }{ 17 | {"returns default when absent", nil, ""}, 18 | {"returns value when present", "value", "value"}, 19 | {"silently ignores unexpected type and returns default", 1, ""}, 20 | } { 21 | t.Run(val.description, func(t *testing.T) { 22 | sk := newStringKey("test") 23 | config := emptyConfig 24 | if val.value != nil { 25 | config = RawConfig{"test": val.value} 26 | } 27 | require.Equal(t, val.expected, sk.pop(config)) 28 | if val.value != nil { 29 | require.Empty(t, config) 30 | } 31 | }) 32 | } 33 | } 34 | 35 | func TestBoolOrStringKey(t *testing.T) { 36 | for _, val := range []struct { 37 | description string 38 | value any 39 | def bool 40 | expected any 41 | err bool 42 | }{ 43 | {"returns true as default when absent", nil, true, true, false}, 44 | {"returns false as default when absent", nil, false, false, false}, 45 | {"returns true when present", true, false, true, false}, 46 | {"returns false when present", false, true, false, false}, 47 | {"returns parsed true string", "true", false, true, false}, 48 | {"returns parsed false string", "false", true, false, false}, 49 | {"fails when invalid string", "", true, false, true}, 50 | {"fails with unexpected type", int64(1), true, false, true}, 51 | } { 52 | t.Run(val.description, func(t *testing.T) { 53 | sk := newBoolOrStringKey("test", val.def) 54 | config := emptyConfig 55 | if val.value != nil { 56 | config = RawConfig{"test": val.value} 57 | } 58 | result, err := sk.pop(config) 59 | if val.value != nil { 60 | require.Empty(t, config) 61 | } 62 | if val.err { 63 | require.Error(t, err) 64 | } else { 65 | require.NoError(t, err) 66 | require.Equal(t, val.expected, result) 67 | } 68 | }) 69 | } 70 | } 71 | 72 | func TestIntOrStringKey(t *testing.T) { 73 | for _, val := range []struct { 74 | description string 75 | value any 76 | def int64 77 | expected any 78 | err bool 79 | }{ 80 | {"returns default when absent", nil, 1, int64(1), false}, 81 | {"returns parsed value of string", "10", 1, int64(10), false}, 82 | {"returns int64 from float", float64(10), 1, int64(10), false}, 83 | {"fails when invalid string", "", 1, int64(1), true}, 84 | {"fails when unexpected type", struct{}{}, 1, int64(1), true}, 85 | } { 86 | t.Run(val.description, func(t *testing.T) { 87 | sk := newIntOrStringKey("test", val.def) 88 | config := emptyConfig 89 | if val.value != nil { 90 | config = RawConfig{"test": val.value} 91 | } 92 | result, err := sk.pop(config) 93 | if val.value != nil { 94 | require.Empty(t, config) 95 | } 96 | if val.err { 97 | require.Error(t, err) 98 | } else { 99 | require.NoError(t, err) 100 | require.Equal(t, val.expected, result) 101 | } 102 | }) 103 | } 104 | } 105 | 106 | func TestMetadataSetKey(t *testing.T) { 107 | input := map[string]any{"k": "v", "k2": "v2"} 108 | invalidInput := map[string]any{"k": 1, "k2": "v2"} 109 | empty := map[string]string{} 110 | notEmpty := map[string]string{"k": "v", "k2": "v2"} 111 | for _, val := range []struct { 112 | description string 113 | value any 114 | def map[string]string 115 | expected map[string]string 116 | expectedWarns bool 117 | err bool 118 | }{ 119 | {"parses pod label string as map", "k=v,k2=v2", empty, notEmpty, false, false}, 120 | {"returns empty map when no labels", "", empty, empty, false, false}, 121 | {"returns empty map when not present", nil, empty, nil, false, false}, 122 | {"supports map as input", input, empty, notEmpty, false, false}, 123 | {"fails on unexpected type", struct{}{}, empty, empty, false, true}, 124 | {"recovers and warns on partially valid string", "k=v,k2", empty, map[string]string{"k": "v"}, true, false}, 125 | {"recovers and warns on invalid map value", invalidInput, empty, map[string]string{"k2": "v2"}, true, false}, 126 | } { 127 | t.Run(val.description, func(t *testing.T) { 128 | k := metadataSetKey("test") 129 | config := emptyConfig 130 | if val.value != nil { 131 | config = RawConfig{"test": val.value} 132 | } 133 | result, warns, err := k.pop(config, "pod", "metadata") 134 | if val.value != nil { 135 | require.Empty(t, config) 136 | } 137 | if val.expectedWarns { 138 | require.NotEmpty(t, warns) 139 | } else { 140 | require.Empty(t, warns) 141 | } 142 | if val.err { 143 | require.Error(t, err) 144 | } else { 145 | require.NoError(t, err) 146 | require.Equal(t, val.expected, result) 147 | } 148 | }) 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /pkg/config/patch.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | 8 | jsonpatch "github.com/evanphx/json-patch" 9 | "k8s.io/apimachinery/pkg/runtime/schema" 10 | kerrors "k8s.io/apimachinery/pkg/util/errors" 11 | "k8s.io/apimachinery/pkg/util/strategicpatch" 12 | utilyaml "k8s.io/apimachinery/pkg/util/yaml" 13 | applymetav1 "k8s.io/client-go/applyconfigurations/meta/v1" 14 | "k8s.io/kubectl/pkg/util/openapi" 15 | 16 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 17 | ) 18 | 19 | const wildcard = "*" 20 | 21 | // ApplyPatches applies a set of patches to an object. 22 | // It returns the number of patches applied, a bool indicating whether there 23 | // were matching patches and the input differed from the output, and any errors 24 | // that occurred. 25 | func ApplyPatches[K any](object, out K, patches []v1alpha1.Patch, resources openapi.Resources) (int, bool, error) { 26 | // marshal object to json for patching 27 | encoded, err := json.Marshal(object) 28 | if err != nil { 29 | return 0, false, fmt.Errorf("error marshalling object to patch: %w", err) 30 | } 31 | 32 | initial := encoded 33 | 34 | // HACK: Unmarshal into TypeMeta to determine `kind` of incoming object. 35 | // The ApplyConfiguration objects don't have any getters, so there's no 36 | // common interface to use to get their `kind`, even though they all have 37 | // a kind field. Golang also doesn't support writing generic functions over 38 | // struct members. This hack can be removed if we add getters to the 39 | // generated applyconfigurations or golang supports this via generics: 40 | // - https://github.com/golang/go/issues/51259 41 | // - https://github.com/golang/go/issues/48522 42 | // - https://github.com/kubernetes/kubernetes/issues/113773 43 | var typeMeta applymetav1.TypeMetaApplyConfiguration 44 | if err := json.Unmarshal(encoded, &typeMeta); err != nil { 45 | return 0, false, fmt.Errorf("unable to extract type info from object for patching: %w", err) 46 | } 47 | 48 | if typeMeta.Kind == nil { 49 | return 0, false, fmt.Errorf("object doesn't specify kind: %v", object) 50 | } 51 | 52 | count := 0 53 | errs := make([]error, 0) 54 | for i, p := range patches { 55 | if p.Kind == *typeMeta.Kind || p.Kind == wildcard { 56 | // determine if the patch is a strategic merge or a json6902 patch 57 | decoder := utilyaml.NewYAMLOrJSONDecoder(bytes.NewReader(p.Patch), 100) 58 | var json6902op jsonpatch.Operation 59 | err = decoder.Decode(&json6902op) 60 | if err != nil { 61 | errs = append(errs, fmt.Errorf("error decoding patch %d: %w", i, err)) 62 | continue 63 | } 64 | 65 | // if there's no operation, it's a strategic merge patch, not 6902 66 | if json6902op.Kind() == "unknown" { 67 | jsonPatch, err := utilyaml.ToJSON(p.Patch) 68 | if err != nil { 69 | errs = append(errs, fmt.Errorf("error converting patch %d to json: %w", i, err)) 70 | continue 71 | } 72 | gv, err := schema.ParseGroupVersion(*typeMeta.APIVersion) 73 | if err != nil { 74 | errs = append(errs, fmt.Errorf("error applying patch %d, to object: %w", i, err)) 75 | continue 76 | } 77 | gvkSchema := resources.LookupResource(gv.WithKind(*typeMeta.Kind)) 78 | patched, err := strategicpatch.StrategicMergePatchUsingLookupPatchMeta(encoded, jsonPatch, strategicpatch.NewPatchMetaFromOpenAPI(gvkSchema)) 79 | if err != nil { 80 | errs = append(errs, fmt.Errorf("error applying patch %d, to object: %w", i, err)) 81 | continue 82 | } 83 | encoded = patched 84 | } else { 85 | json6902patch := jsonpatch.Patch([]jsonpatch.Operation{json6902op}) 86 | patched, err := json6902patch.Apply(encoded) 87 | if err != nil { 88 | errs = append(errs, fmt.Errorf("error applying patch %d to object: %w", i, err)) 89 | continue 90 | } 91 | encoded = patched 92 | } 93 | count++ 94 | } 95 | } 96 | 97 | if err := json.Unmarshal(encoded, out); err != nil { 98 | errs = append(errs, fmt.Errorf("error converting back to object: %w", err)) 99 | } 100 | 101 | diff := false 102 | if count > 0 { 103 | // return true if there were patches defined and the output bytes differ 104 | diff = !bytes.Equal(encoded, initial) 105 | } 106 | return count, diff, kerrors.NewAggregate(errs) 107 | } 108 | -------------------------------------------------------------------------------- /pkg/controller/check_migrations.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/exp/slices" 7 | corev1 "k8s.io/api/core/v1" 8 | "k8s.io/client-go/tools/record" 9 | 10 | "github.com/authzed/controller-idioms/handler" 11 | "github.com/authzed/controller-idioms/hash" 12 | 13 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 14 | "github.com/authzed/spicedb-operator/pkg/metadata" 15 | ) 16 | 17 | const ( 18 | EventRunningMigrations = "RunningMigrations" 19 | 20 | HandlerDeploymentKey handler.Key = "deploymentChain" 21 | HandlerMigrationRunKey handler.Key = "runMigration" 22 | HandlerWaitForMigrationsKey handler.Key = "waitForMigrationChain" 23 | ) 24 | 25 | type MigrationCheckHandler struct { 26 | recorder record.EventRecorder 27 | 28 | nextMigrationRunHandler handler.ContextHandler 29 | nextWaitForJobHandler handler.ContextHandler 30 | nextDeploymentHandler handler.ContextHandler 31 | } 32 | 33 | func (m *MigrationCheckHandler) Handle(ctx context.Context) { 34 | migrationHash := CtxMigrationHash.MustValue(ctx) 35 | 36 | hasJob := false 37 | hasDeployment := false 38 | for _, d := range CtxDeployments.MustValue(ctx) { 39 | if d.Annotations != nil && hash.SecureEqual(d.Annotations[metadata.SpiceDBMigrationRequirementsKey], migrationHash) { 40 | hasDeployment = true 41 | break 42 | } 43 | } 44 | for _, j := range CtxJobs.MustValue(ctx) { 45 | if j.Annotations != nil && hash.SecureEqual(j.Annotations[metadata.SpiceDBMigrationRequirementsKey], migrationHash) { 46 | hasJob = true 47 | ctx = CtxCurrentMigrationJob.WithValue(ctx, j) 48 | break 49 | } 50 | } 51 | 52 | // don't handle migrations at all if `skipMigrations` is set, if the 53 | // `memory` datastore is used, or if the update graph says there are no 54 | // migrations for this step. 55 | config := CtxConfig.MustValue(ctx) 56 | if config.SkipMigrations || config.DatastoreEngine == "memory" { 57 | m.nextDeploymentHandler.Handle(ctx) 58 | return 59 | } 60 | status := CtxCluster.MustValue(ctx).Status 61 | if status.CurrentVersion != nil && !slices.Contains(status.CurrentVersion.Attributes, v1alpha1.SpiceDBVersionAttributesMigration) { 62 | m.nextDeploymentHandler.Handle(ctx) 63 | return 64 | } 65 | 66 | // if there's no job and no (updated) deployment, create the job 67 | if !hasDeployment && !hasJob { 68 | m.recorder.Eventf(CtxCluster.MustValue(ctx), corev1.EventTypeNormal, EventRunningMigrations, "Running migration job for %s", CtxConfig.MustValue(ctx).TargetSpiceDBImage) 69 | m.nextMigrationRunHandler.Handle(ctx) 70 | return 71 | } 72 | 73 | // if there's a job but no (updated) deployment, wait for the job 74 | if hasJob && !hasDeployment { 75 | m.nextWaitForJobHandler.Handle(ctx) 76 | return 77 | } 78 | 79 | // if the deployment is up to date, continue 80 | m.nextDeploymentHandler.Handle(ctx) 81 | } 82 | -------------------------------------------------------------------------------- /pkg/controller/check_migrations_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | appsv1 "k8s.io/api/apps/v1" 9 | batchv1 "k8s.io/api/batch/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/client-go/tools/record" 12 | 13 | "github.com/authzed/controller-idioms/handler" 14 | "github.com/authzed/controller-idioms/queue/fake" 15 | 16 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 17 | "github.com/authzed/spicedb-operator/pkg/config" 18 | "github.com/authzed/spicedb-operator/pkg/metadata" 19 | ) 20 | 21 | func TestCheckMigrationsHandler(t *testing.T) { 22 | tests := []struct { 23 | name string 24 | 25 | config config.Config 26 | migrationHash string 27 | existingJobs []*batchv1.Job 28 | existingDeployments []*appsv1.Deployment 29 | 30 | expectEvents []string 31 | expectRequeueErr error 32 | expectNext handler.Key 33 | }{ 34 | { 35 | name: "run migrations if no job, no deployment", 36 | config: config.Config{MigrationConfig: config.MigrationConfig{TargetSpiceDBImage: "test"}}, 37 | migrationHash: "hash", 38 | existingJobs: []*batchv1.Job{}, 39 | existingDeployments: []*appsv1.Deployment{}, 40 | expectEvents: []string{"Normal RunningMigrations Running migration job for test"}, 41 | expectNext: HandlerMigrationRunKey, 42 | }, 43 | { 44 | name: "run migration if non-matching job and no deployment", 45 | migrationHash: "hash", 46 | config: config.Config{MigrationConfig: config.MigrationConfig{TargetSpiceDBImage: "test"}}, 47 | existingJobs: []*batchv1.Job{{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 48 | metadata.SpiceDBMigrationRequirementsKey: "nope", 49 | }}}}, 50 | existingDeployments: []*appsv1.Deployment{}, 51 | expectEvents: []string{"Normal RunningMigrations Running migration job for test"}, 52 | expectNext: HandlerMigrationRunKey, 53 | }, 54 | { 55 | name: "wait for migrations if matching job but no deployment", 56 | config: config.Config{MigrationConfig: config.MigrationConfig{TargetSpiceDBImage: "test"}}, 57 | migrationHash: "hash", 58 | existingJobs: []*batchv1.Job{ 59 | { 60 | ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 61 | metadata.SpiceDBMigrationRequirementsKey: "hash", 62 | }}, 63 | }, { 64 | ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 65 | metadata.SpiceDBMigrationRequirementsKey: "nope", 66 | }}, 67 | }, 68 | }, 69 | existingDeployments: []*appsv1.Deployment{}, 70 | expectNext: HandlerWaitForMigrationsKey, 71 | }, 72 | { 73 | name: "check deployment if skipMigrations = true", 74 | config: config.Config{SpiceConfig: config.SpiceConfig{SkipMigrations: true}}, 75 | existingDeployments: []*appsv1.Deployment{{}}, 76 | expectNext: HandlerDeploymentKey, 77 | }, 78 | { 79 | name: "check deployment if deployment is up to date", 80 | migrationHash: "hash", 81 | config: config.Config{MigrationConfig: config.MigrationConfig{TargetSpiceDBImage: "test"}}, 82 | existingDeployments: []*appsv1.Deployment{{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 83 | metadata.SpiceDBMigrationRequirementsKey: "hash", 84 | }}}}, 85 | expectNext: HandlerDeploymentKey, 86 | }, 87 | { 88 | name: "check deployment if deployment is up to date job is up to date", 89 | config: config.Config{MigrationConfig: config.MigrationConfig{TargetSpiceDBImage: "test"}}, 90 | migrationHash: "hash", 91 | existingJobs: []*batchv1.Job{{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 92 | metadata.SpiceDBMigrationRequirementsKey: "hash", 93 | }}}}, 94 | existingDeployments: []*appsv1.Deployment{{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 95 | metadata.SpiceDBMigrationRequirementsKey: "hash", 96 | }}}}, 97 | expectNext: HandlerDeploymentKey, 98 | }, 99 | } 100 | for _, tt := range tests { 101 | t.Run(tt.name, func(t *testing.T) { 102 | tt := tt 103 | ctrls := &fake.FakeInterface{} 104 | 105 | ctx := CtxConfig.WithValue(context.Background(), &tt.config) 106 | ctx = QueueOps.WithValue(ctx, ctrls) 107 | ctx = CtxCluster.WithValue(ctx, &v1alpha1.SpiceDBCluster{}) 108 | ctx = CtxCluster.WithValue(ctx, &v1alpha1.SpiceDBCluster{}) 109 | ctx = CtxJobs.WithValue(ctx, tt.existingJobs) 110 | ctx = CtxDeployments.WithValue(ctx, tt.existingDeployments) 111 | ctx = CtxMigrationHash.WithValue(ctx, "hash") 112 | 113 | recorder := record.NewFakeRecorder(1) 114 | 115 | var called handler.Key 116 | h := &MigrationCheckHandler{ 117 | recorder: recorder, 118 | nextDeploymentHandler: handler.ContextHandlerFunc(func(_ context.Context) { 119 | called = HandlerDeploymentKey 120 | }), 121 | nextWaitForJobHandler: handler.ContextHandlerFunc(func(_ context.Context) { 122 | called = HandlerWaitForMigrationsKey 123 | }), 124 | nextMigrationRunHandler: handler.ContextHandlerFunc(func(_ context.Context) { 125 | called = HandlerMigrationRunKey 126 | }), 127 | } 128 | h.Handle(ctx) 129 | 130 | require.Equal(t, tt.expectNext, called) 131 | ExpectEvents(t, recorder, tt.expectEvents) 132 | 133 | if tt.expectRequeueErr != nil { 134 | require.Equal(t, 1, ctrls.RequeueErrCallCount()) 135 | require.Equal(t, tt.expectRequeueErr, ctrls.RequeueErrArgsForCall(0)) 136 | } 137 | }) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /pkg/controller/cleanup_job.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | batchv1 "k8s.io/api/batch/v1" 7 | corev1 "k8s.io/api/core/v1" 8 | "k8s.io/apimachinery/pkg/types" 9 | 10 | "github.com/authzed/controller-idioms/hash" 11 | "github.com/authzed/controller-idioms/typed" 12 | 13 | "github.com/authzed/spicedb-operator/pkg/metadata" 14 | ) 15 | 16 | type JobCleanupHandler struct { 17 | registry *typed.Registry 18 | getJobPods func(ctx context.Context) []*corev1.Pod 19 | getJobs func(ctx context.Context) []*batchv1.Job 20 | deleteJob func(ctx context.Context, nn types.NamespacedName) error 21 | deletePod func(ctx context.Context, nn types.NamespacedName) error 22 | } 23 | 24 | func (s *JobCleanupHandler) Handle(ctx context.Context) { 25 | jobs := s.getJobs(ctx) 26 | pods := s.getJobPods(ctx) 27 | deployment := *CtxCurrentSpiceDeployment.MustValue(ctx) 28 | if deployment.Annotations == nil || len(jobs)+len(pods) == 0 { 29 | QueueOps.Done(ctx) 30 | return 31 | } 32 | 33 | jobNames := make(map[string]struct{}) 34 | for _, j := range jobs { 35 | jobNames[j.Name] = struct{}{} 36 | if j.Annotations == nil { 37 | continue 38 | } 39 | if hash.SecureEqual( 40 | j.Annotations[metadata.SpiceDBMigrationRequirementsKey], 41 | deployment.Annotations[metadata.SpiceDBMigrationRequirementsKey]) && 42 | jobConditionHasStatus(j, batchv1.JobComplete, corev1.ConditionTrue) { 43 | if err := s.deleteJob(ctx, types.NamespacedName{ 44 | Namespace: j.GetNamespace(), 45 | Name: j.GetName(), 46 | }); err != nil { 47 | QueueOps.RequeueAPIErr(ctx, err) 48 | return 49 | } 50 | } 51 | } 52 | 53 | for _, p := range pods { 54 | labels := p.GetLabels() 55 | if labels == nil { 56 | continue 57 | } 58 | jobName, ok := labels["job-name"] 59 | if !ok { 60 | continue 61 | } 62 | if _, ok := jobNames[jobName]; ok { 63 | // job still exists 64 | QueueOps.Requeue(ctx) 65 | return 66 | } 67 | 68 | if err := s.deletePod(ctx, types.NamespacedName{ 69 | Namespace: p.GetNamespace(), 70 | Name: p.GetName(), 71 | }); err != nil { 72 | QueueOps.RequeueAPIErr(ctx, err) 73 | return 74 | } 75 | } 76 | 77 | QueueOps.Done(ctx) 78 | } 79 | -------------------------------------------------------------------------------- /pkg/controller/cleanup_job_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | appsv1 "k8s.io/api/apps/v1" 9 | batchv1 "k8s.io/api/batch/v1" 10 | corev1 "k8s.io/api/core/v1" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/apimachinery/pkg/types" 13 | 14 | "github.com/authzed/controller-idioms/queue/fake" 15 | 16 | "github.com/authzed/spicedb-operator/pkg/metadata" 17 | ) 18 | 19 | func TestCleanupJobsHandler(t *testing.T) { 20 | tests := []struct { 21 | name string 22 | 23 | existingDeployment *appsv1.Deployment 24 | existingJobPods []*corev1.Pod 25 | existingJobs []*batchv1.Job 26 | 27 | expectDeletedPods []string 28 | expectDeletedJobs []string 29 | expectRequeueErr error 30 | expectRequeue bool 31 | expectDone bool 32 | }{ 33 | { 34 | name: "don't clean up if deployment has no annotations", 35 | existingDeployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Annotations: nil}}, 36 | existingJobs: []*batchv1.Job{{}}, 37 | existingJobPods: []*corev1.Pod{{}}, 38 | expectDone: true, 39 | }, 40 | { 41 | name: "don't clean up if no jobs and no pods", 42 | existingDeployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{"an": "annotation"}}}, 43 | existingJobs: []*batchv1.Job{}, 44 | existingJobPods: []*corev1.Pod{}, 45 | expectDone: true, 46 | }, 47 | { 48 | name: "deletes completed jobs with matching migration hash", 49 | existingDeployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 50 | metadata.SpiceDBMigrationRequirementsKey: "test", 51 | }}}, 52 | existingJobs: []*batchv1.Job{{ 53 | ObjectMeta: metav1.ObjectMeta{ 54 | Name: "mjob", 55 | Annotations: map[string]string{ 56 | metadata.SpiceDBMigrationRequirementsKey: "test", 57 | }, 58 | }, 59 | Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{ 60 | Type: batchv1.JobComplete, 61 | Status: corev1.ConditionTrue, 62 | }}}, 63 | }}, 64 | expectDeletedJobs: []string{"mjob"}, 65 | expectDone: true, 66 | }, 67 | { 68 | name: "doesn't delete completed jobs without migration hash", 69 | existingDeployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 70 | metadata.SpiceDBMigrationRequirementsKey: "test", 71 | }}}, 72 | existingJobs: []*batchv1.Job{{ 73 | ObjectMeta: metav1.ObjectMeta{ 74 | Name: "mjob", 75 | Annotations: map[string]string{ 76 | metadata.SpiceDBMigrationRequirementsKey: "different", 77 | }, 78 | }, 79 | Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{ 80 | Type: batchv1.JobComplete, 81 | Status: corev1.ConditionTrue, 82 | }}}, 83 | }}, 84 | expectDone: true, 85 | }, 86 | { 87 | name: "doesn't delete incomplete jobs with matching migration hash", 88 | existingDeployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 89 | metadata.SpiceDBMigrationRequirementsKey: "test", 90 | }}}, 91 | existingJobs: []*batchv1.Job{{ 92 | ObjectMeta: metav1.ObjectMeta{ 93 | Name: "mjob", 94 | Annotations: map[string]string{ 95 | metadata.SpiceDBMigrationRequirementsKey: "test", 96 | }, 97 | }, 98 | Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{ 99 | Type: batchv1.JobComplete, 100 | Status: corev1.ConditionFalse, 101 | }}}, 102 | }}, 103 | expectDone: true, 104 | }, 105 | { 106 | name: "deletes pods with missing jobs", 107 | existingDeployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{"a": "test"}}}, 108 | existingJobPods: []*corev1.Pod{{ 109 | ObjectMeta: metav1.ObjectMeta{ 110 | Name: "mpod", 111 | Labels: map[string]string{ 112 | "job-name": "mjob", 113 | }, 114 | }, 115 | }}, 116 | expectDeletedPods: []string{"mpod"}, 117 | expectDone: true, 118 | }, 119 | { 120 | name: "doesn't delete pods with extant jobs", 121 | existingDeployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{"a": "test"}}}, 122 | existingJobs: []*batchv1.Job{{ 123 | ObjectMeta: metav1.ObjectMeta{ 124 | Name: "mjob", 125 | Annotations: map[string]string{ 126 | metadata.SpiceDBMigrationRequirementsKey: "test", 127 | }, 128 | }, 129 | Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{ 130 | Type: batchv1.JobComplete, 131 | Status: corev1.ConditionFalse, 132 | }}}, 133 | }}, 134 | existingJobPods: []*corev1.Pod{{ 135 | ObjectMeta: metav1.ObjectMeta{ 136 | Name: "mpod", 137 | Labels: map[string]string{ 138 | "job-name": "mjob", 139 | }, 140 | }, 141 | }}, 142 | expectRequeue: true, 143 | }, 144 | } 145 | for _, tt := range tests { 146 | t.Run(tt.name, func(t *testing.T) { 147 | ctrls := &fake.FakeInterface{} 148 | 149 | ctx := context.Background() 150 | ctx = QueueOps.WithValue(ctx, ctrls) 151 | ctx = CtxCurrentSpiceDeployment.WithValue(ctx, tt.existingDeployment) 152 | 153 | if tt.expectDeletedJobs == nil { 154 | tt.expectDeletedJobs = make([]string, 0) 155 | } 156 | if tt.expectDeletedPods == nil { 157 | tt.expectDeletedPods = make([]string, 0) 158 | } 159 | deletedPods := make([]string, 0) 160 | deletedJobs := make([]string, 0) 161 | h := &JobCleanupHandler{ 162 | getJobs: func(_ context.Context) []*batchv1.Job { 163 | return tt.existingJobs 164 | }, 165 | getJobPods: func(_ context.Context) []*corev1.Pod { 166 | return tt.existingJobPods 167 | }, 168 | deletePod: func(_ context.Context, nn types.NamespacedName) error { 169 | deletedPods = append(deletedPods, nn.Name) 170 | return nil 171 | }, 172 | deleteJob: func(_ context.Context, nn types.NamespacedName) error { 173 | deletedJobs = append(deletedJobs, nn.Name) 174 | return nil 175 | }, 176 | } 177 | h.Handle(ctx) 178 | 179 | require.Equal(t, tt.expectRequeue, ctrls.RequeueCallCount() == 1) 180 | require.Equal(t, tt.expectDone, ctrls.DoneCallCount() == 1) 181 | require.Equal(t, tt.expectDeletedJobs, deletedJobs) 182 | require.Equal(t, tt.expectDeletedPods, deletedPods) 183 | if tt.expectRequeueErr != nil { 184 | require.Equal(t, 1, ctrls.RequeueErrCallCount()) 185 | require.Equal(t, tt.expectRequeueErr, ctrls.RequeueErrArgsForCall(0)) 186 | } 187 | }) 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /pkg/controller/client.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | 7 | "k8s.io/apimachinery/pkg/types" 8 | 9 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 10 | "github.com/authzed/spicedb-operator/pkg/metadata" 11 | ) 12 | 13 | func (c *Controller) PatchStatus(ctx context.Context, patch *v1alpha1.SpiceDBCluster) error { 14 | for _, c := range patch.Status.Conditions { 15 | c.ObservedGeneration = patch.Generation 16 | } 17 | patch.ManagedFields = nil 18 | patch.Status.ObservedGeneration = patch.Generation 19 | data, err := json.Marshal(patch) 20 | if err != nil { 21 | return err 22 | } 23 | _, err = c.client.Resource(v1alpha1ClusterGVR).Namespace(patch.Namespace).Patch(ctx, patch.Name, types.ApplyPatchType, data, metadata.PatchForceOwned, "status") 24 | return err 25 | } 26 | 27 | func (c *Controller) Patch(ctx context.Context, patch *v1alpha1.SpiceDBCluster) error { 28 | data, err := json.Marshal(patch) 29 | if err != nil { 30 | return err 31 | } 32 | _, err = c.client.Resource(v1alpha1ClusterGVR).Namespace(patch.Namespace).Patch(ctx, patch.Name, types.ApplyPatchType, data, metadata.PatchForceOwned) 33 | return err 34 | } 35 | -------------------------------------------------------------------------------- /pkg/controller/config_change.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/authzed/controller-idioms/handler" 7 | "github.com/authzed/controller-idioms/hash" 8 | "github.com/go-logr/logr" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | 11 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 12 | ) 13 | 14 | type ConfigChangedHandler struct { 15 | patchStatus func(ctx context.Context, patch *v1alpha1.SpiceDBCluster) error 16 | next handler.ContextHandler 17 | } 18 | 19 | func (c *ConfigChangedHandler) Handle(ctx context.Context) { 20 | cluster := CtxCluster.MustValue(ctx) 21 | secret := CtxSecret.Value(ctx) 22 | var secretHash string 23 | if secret != nil { 24 | secretHash = hash.SecureObject(secret.Data) 25 | ctx = CtxSecretHash.WithValue(ctx, secretHash) 26 | } 27 | status := &v1alpha1.SpiceDBCluster{ 28 | TypeMeta: metav1.TypeMeta{ 29 | Kind: v1alpha1.SpiceDBClusterKind, 30 | APIVersion: v1alpha1.SchemeGroupVersion.String(), 31 | }, 32 | ObjectMeta: metav1.ObjectMeta{Namespace: cluster.Namespace, Name: cluster.Name, Generation: cluster.Generation}, 33 | Status: *cluster.Status.DeepCopy(), 34 | } 35 | 36 | preconditionsFailedCondition := cluster.FindStatusCondition(v1alpha1.ConditionTypePreconditionsFailed) 37 | if cluster.GetGeneration() != status.Status.ObservedGeneration || secretHash != status.Status.SecretHash || 38 | (preconditionsFailedCondition != nil && preconditionsFailedCondition.Reason == v1alpha1.ConditionReasonMissingSecret) { 39 | logr.FromContextOrDiscard(ctx).V(4).Info("spicedb configuration changed") 40 | status.Status.ObservedGeneration = cluster.GetGeneration() 41 | status.Status.SecretHash = secretHash 42 | status.RemoveStatusCondition(v1alpha1.ConditionTypePreconditionsFailed) 43 | status.SetStatusCondition(v1alpha1.NewValidatingConfigCondition(secretHash)) 44 | if err := c.patchStatus(ctx, status); err != nil { 45 | QueueOps.RequeueAPIErr(ctx, err) 46 | return 47 | } 48 | } 49 | cluster.Status = status.Status 50 | ctx = CtxCluster.WithValue(ctx, cluster) 51 | c.next.Handle(ctx) 52 | } 53 | -------------------------------------------------------------------------------- /pkg/controller/context.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | appsv1 "k8s.io/api/apps/v1" 5 | batchv1 "k8s.io/api/batch/v1" 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/apimachinery/pkg/types" 8 | 9 | "github.com/authzed/controller-idioms/queue" 10 | "github.com/authzed/controller-idioms/typedctx" 11 | 12 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 13 | "github.com/authzed/spicedb-operator/pkg/config" 14 | ) 15 | 16 | var ( 17 | QueueOps = queue.NewQueueOperationsCtx() 18 | CtxOperatorConfig = typedctx.WithDefault[*config.OperatorConfig](nil) 19 | CtxCacheNamespace = typedctx.WithDefault("") 20 | CtxClusterNN = typedctx.WithDefault(types.NamespacedName{}) 21 | CtxSecretNN = typedctx.WithDefault(types.NamespacedName{}) 22 | CtxSecret = typedctx.WithDefault[*corev1.Secret](nil) 23 | CtxSecretHash = typedctx.WithDefault("") 24 | CtxCluster = typedctx.WithDefault[*v1alpha1.SpiceDBCluster](nil) 25 | CtxConfig = typedctx.WithDefault[*config.Config](nil) 26 | CtxMigrationHash = typedctx.WithDefault("") 27 | CtxDeployments = typedctx.Boxed(make([]*appsv1.Deployment, 0)) 28 | CtxJobs = typedctx.Boxed(make([]*batchv1.Job, 0)) 29 | CtxCurrentMigrationJob = typedctx.WithDefault[*batchv1.Job](nil) 30 | CtxCurrentSpiceDeployment = typedctx.WithDefault[*appsv1.Deployment](nil) 31 | CtxSelfPauseObject = typedctx.WithDefault(new(v1alpha1.SpiceDBCluster)) 32 | ) 33 | -------------------------------------------------------------------------------- /pkg/controller/controller_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "github.com/authzed/controller-idioms/typed" 10 | "github.com/stretchr/testify/require" 11 | corev1 "k8s.io/api/core/v1" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | "k8s.io/client-go/dynamic/fake" 14 | kfake "k8s.io/client-go/kubernetes/fake" 15 | "k8s.io/client-go/kubernetes/scheme" 16 | "k8s.io/client-go/tools/record" 17 | "k8s.io/client-go/util/workqueue" 18 | 19 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 20 | "github.com/authzed/spicedb-operator/pkg/metadata" 21 | ) 22 | 23 | type keyRecordingQueue struct { 24 | workqueue.TypedRateLimitingInterface[any] 25 | Items chan any 26 | } 27 | 28 | func newKeyRecordingQueue(queue workqueue.TypedRateLimitingInterface[any]) *keyRecordingQueue { 29 | return &keyRecordingQueue{ 30 | TypedRateLimitingInterface: queue, 31 | Items: make(chan any), 32 | } 33 | } 34 | 35 | func (q *keyRecordingQueue) Add(item any) { 36 | q.Items <- item 37 | q.TypedRateLimitingInterface.Add(item) 38 | } 39 | 40 | func (q *keyRecordingQueue) AddAfter(item any, d time.Duration) { 41 | q.Items <- item 42 | q.TypedRateLimitingInterface.AddAfter(item, d) 43 | } 44 | 45 | func (q *keyRecordingQueue) AddRateLimited(item any) { 46 | q.Items <- item 47 | q.TypedRateLimitingInterface.AddRateLimited(item) 48 | } 49 | 50 | func TestControllerNamespacing(t *testing.T) { 51 | tests := []struct { 52 | name string 53 | watchedNamespaces []string 54 | createNamespaces []string 55 | spiceDBClusters map[string]string 56 | services map[string]string 57 | expectedKeys []string 58 | }{ 59 | { 60 | name: "default to all namespaces", 61 | watchedNamespaces: nil, 62 | createNamespaces: []string{"test", "test2", "test3"}, 63 | spiceDBClusters: map[string]string{"test3": "test3"}, 64 | services: map[string]string{ 65 | "test": "test", 66 | "test2": "test2", 67 | }, 68 | expectedKeys: []string{ 69 | "spicedbclusters.v1alpha1.authzed.com::test/test", 70 | "spicedbclusters.v1alpha1.authzed.com::test2/test2", 71 | "spicedbclusters.v1alpha1.authzed.com::test3/test3", 72 | }, 73 | }, 74 | { 75 | name: "explicitly watch all namespaces", 76 | watchedNamespaces: []string{""}, 77 | createNamespaces: []string{"test", "test2", "test3"}, 78 | spiceDBClusters: map[string]string{"test3": "test3"}, 79 | services: map[string]string{ 80 | "test": "test", 81 | "test2": "test2", 82 | }, 83 | expectedKeys: []string{ 84 | "spicedbclusters.v1alpha1.authzed.com::test/test", 85 | "spicedbclusters.v1alpha1.authzed.com::test2/test2", 86 | "spicedbclusters.v1alpha1.authzed.com::test3/test3", 87 | }, 88 | }, 89 | { 90 | name: "watch one namespace (owned objects)", 91 | watchedNamespaces: []string{"test"}, 92 | createNamespaces: []string{"test", "test2", "test3"}, 93 | spiceDBClusters: map[string]string{"test3": "test3"}, 94 | services: map[string]string{ 95 | "test": "test", 96 | "test2": "test2", 97 | }, 98 | expectedKeys: []string{ 99 | "spicedbclusters.v1alpha1.authzed.com::test/test", 100 | }, 101 | }, 102 | { 103 | name: "watch one namespace (external objects)", 104 | watchedNamespaces: []string{"test2"}, 105 | createNamespaces: []string{"test", "test2", "test3"}, 106 | spiceDBClusters: map[string]string{"test3": "test3"}, 107 | services: map[string]string{ 108 | "test": "test", 109 | "test2": "test2", 110 | }, 111 | expectedKeys: []string{ 112 | "spicedbclusters.v1alpha1.authzed.com::test2/test2", 113 | }, 114 | }, 115 | { 116 | name: "watch multiple namespaces", 117 | watchedNamespaces: []string{"test2", "test3"}, 118 | createNamespaces: []string{"test", "test2", "test3"}, 119 | spiceDBClusters: map[string]string{"test3": "test3"}, 120 | services: map[string]string{ 121 | "test": "test", 122 | "test2": "test2", 123 | }, 124 | expectedKeys: []string{ 125 | "spicedbclusters.v1alpha1.authzed.com::test2/test2", 126 | "spicedbclusters.v1alpha1.authzed.com::test3/test3", 127 | }, 128 | }, 129 | } 130 | for _, tt := range tests { 131 | t.Run(tt.name, func(t *testing.T) { 132 | ctx, cancel := context.WithCancel(context.Background()) 133 | t.Cleanup(cancel) 134 | registry := typed.NewRegistry() 135 | broadcaster := record.NewBroadcaster() 136 | dclient := fake.NewSimpleDynamicClient(scheme.Scheme) 137 | kclient := kfake.NewSimpleClientset() 138 | c, err := NewController(ctx, registry, dclient, kclient, nil, "", broadcaster, tt.watchedNamespaces) 139 | require.NoError(t, err) 140 | queue := newKeyRecordingQueue(workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any]())) 141 | c.Queue = queue 142 | go c.Start(ctx, 1) 143 | 144 | for _, ns := range tt.createNamespaces { 145 | ns, err := typed.ObjToUnstructuredObj(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}) 146 | require.NoError(t, err) 147 | _, err = dclient.Resource(corev1.SchemeGroupVersion.WithResource("namespaces")).Create(ctx, ns, metav1.CreateOptions{}) 148 | require.NoError(t, err) 149 | } 150 | 151 | // test that non-owned objects (i.e. services) are watched in 152 | // the appropriate namespaces as well 153 | for ns, spicedb := range tt.services { 154 | service, err := typed.ObjToUnstructuredObj(&corev1.Service{ 155 | ObjectMeta: metav1.ObjectMeta{ 156 | Name: "test", 157 | Namespace: ns, 158 | Labels: map[string]string{ 159 | metadata.OperatorManagedLabelKey: metadata.OperatorManagedLabelValue, 160 | }, 161 | Annotations: map[string]string{ 162 | metadata.OwnerAnnotationKeyPrefix + spicedb: "owned", 163 | }, 164 | }, 165 | }) 166 | require.NoError(t, err) 167 | 168 | _, err = dclient.Resource(corev1.SchemeGroupVersion.WithResource("services")).Namespace(ns).Create(ctx, service, metav1.CreateOptions{}) 169 | require.NoError(t, err) 170 | } 171 | 172 | for ns, spicedb := range tt.spiceDBClusters { 173 | service, err := typed.ObjToUnstructuredObj(&v1alpha1.SpiceDBCluster{ 174 | ObjectMeta: metav1.ObjectMeta{ 175 | Name: spicedb, 176 | Namespace: ns, 177 | }, 178 | }) 179 | require.NoError(t, err) 180 | 181 | _, err = dclient.Resource(v1alpha1ClusterGVR).Namespace(ns).Create(ctx, service, metav1.CreateOptions{}) 182 | require.NoError(t, err) 183 | } 184 | 185 | gotKeys := make([]any, 0) 186 | var wg sync.WaitGroup 187 | wg.Add(1) 188 | go func() { 189 | defer wg.Done() 190 | for { 191 | select { 192 | case s := <-queue.Items: 193 | gotKeys = append(gotKeys, s) 194 | if len(gotKeys) == len(tt.expectedKeys) { 195 | return 196 | } 197 | case <-ctx.Done(): 198 | return 199 | } 200 | } 201 | }() 202 | wg.Wait() 203 | require.ElementsMatch(t, tt.expectedKeys, gotKeys) 204 | }) 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /pkg/controller/ensure_deployment.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sort" 7 | "time" 8 | 9 | appsv1 "k8s.io/api/apps/v1" 10 | corev1 "k8s.io/api/core/v1" 11 | "k8s.io/apimachinery/pkg/types" 12 | applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1" 13 | 14 | "github.com/authzed/controller-idioms/handler" 15 | "github.com/authzed/controller-idioms/hash" 16 | 17 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 18 | "github.com/authzed/spicedb-operator/pkg/metadata" 19 | ) 20 | 21 | type DeploymentHandler struct { 22 | applyDeployment func(ctx context.Context, dep *applyappsv1.DeploymentApplyConfiguration) (*appsv1.Deployment, error) 23 | deleteDeployment func(ctx context.Context, nn types.NamespacedName) error 24 | getDeploymentPods func(ctx context.Context) []*corev1.Pod 25 | patchStatus func(ctx context.Context, patch *v1alpha1.SpiceDBCluster) error 26 | next handler.ContextHandler 27 | } 28 | 29 | func (m *DeploymentHandler) Handle(ctx context.Context) { 30 | // TODO: unconditional status change can be a separate handler 31 | currentStatus := CtxCluster.MustValue(ctx) 32 | // remove migrating condition if present and set the current migration hash 33 | if currentStatus.FindStatusCondition(v1alpha1.ConditionTypeMigrating) != nil || 34 | currentStatus.Status.CurrentMigrationHash != currentStatus.Status.TargetMigrationHash { 35 | currentStatus.RemoveStatusCondition(v1alpha1.ConditionTypeMigrating) 36 | currentStatus.Status.CurrentMigrationHash = currentStatus.Status.TargetMigrationHash 37 | currentStatus.SetStatusCondition(v1alpha1.NewRollingCondition("Rolling deployment to latest version")) 38 | if err := m.patchStatus(ctx, currentStatus); err != nil { 39 | QueueOps.RequeueAPIErr(ctx, err) 40 | return 41 | } 42 | ctx = CtxCluster.WithValue(ctx, currentStatus) 43 | } 44 | 45 | migrationHash := CtxMigrationHash.MustValue(ctx) 46 | secretHash := CtxSecretHash.MustValue(ctx) 47 | config := CtxConfig.MustValue(ctx) 48 | newDeployment := config.Deployment(migrationHash, secretHash) 49 | deploymentHash := hash.Object(newDeployment) 50 | 51 | matchingObjs := make([]*appsv1.Deployment, 0) 52 | extraObjs := make([]*appsv1.Deployment, 0) 53 | for _, o := range CtxDeployments.MustValue(ctx) { 54 | annotations := o.GetAnnotations() 55 | if annotations == nil { 56 | extraObjs = append(extraObjs, o) 57 | } 58 | if hash.Equal(annotations[metadata.SpiceDBConfigKey], deploymentHash) { 59 | matchingObjs = append(matchingObjs, o) 60 | } else { 61 | extraObjs = append(extraObjs, o) 62 | } 63 | } 64 | 65 | var cachedDeployment *appsv1.Deployment 66 | // deployment with correct hash exists 67 | if len(matchingObjs) == 1 { 68 | cachedDeployment = matchingObjs[0] 69 | ctx = CtxCurrentSpiceDeployment.WithValue(ctx, cachedDeployment) 70 | 71 | // delete extra objects 72 | for _, o := range extraObjs { 73 | if err := m.deleteDeployment(ctx, types.NamespacedName{Namespace: currentStatus.Namespace, Name: o.GetName()}); err != nil { 74 | QueueOps.RequeueAPIErr(ctx, err) 75 | return 76 | } 77 | } 78 | } 79 | 80 | // apply if no matching object in controller 81 | if len(matchingObjs) == 0 { 82 | deployment, err := m.applyDeployment(ctx, 83 | newDeployment.WithAnnotations( 84 | map[string]string{metadata.SpiceDBConfigKey: deploymentHash}, 85 | ), 86 | ) 87 | if err != nil { 88 | QueueOps.RequeueAPIErr(ctx, err) 89 | return 90 | } 91 | ctx = CtxCurrentSpiceDeployment.WithValue(ctx, deployment) 92 | } 93 | 94 | // if the deployment isn't in the cache yet, wait until another event 95 | // comes in for it 96 | if cachedDeployment == nil { 97 | QueueOps.RequeueAfter(ctx, time.Second) 98 | return 99 | } 100 | 101 | // check if any pods have errors 102 | if cachedDeployment.Status.UnavailableReplicas > 0 { 103 | // sort pods by newest first 104 | pods := m.getDeploymentPods(ctx) 105 | sort.Slice(pods, func(i, j int) bool { 106 | return pods[i].CreationTimestamp.Before(&pods[j].CreationTimestamp) 107 | }) 108 | for _, p := range m.getDeploymentPods(ctx) { 109 | for _, s := range p.Status.ContainerStatuses { 110 | if s.LastTerminationState.Terminated != nil { 111 | currentStatus.SetStatusCondition(v1alpha1.NewPodErrorCondition(s.LastTerminationState.Terminated.Message)) 112 | if err := m.patchStatus(ctx, currentStatus); err != nil { 113 | QueueOps.RequeueAPIErr(ctx, err) 114 | return 115 | } 116 | QueueOps.RequeueAfter(ctx, 2*time.Second) 117 | return 118 | } 119 | } 120 | } 121 | } 122 | 123 | // wait for deployment to be available 124 | if cachedDeployment.Status.AvailableReplicas != config.Replicas || 125 | cachedDeployment.Status.ReadyReplicas != config.Replicas || 126 | cachedDeployment.Status.UpdatedReplicas != config.Replicas || 127 | cachedDeployment.Status.ObservedGeneration != cachedDeployment.Generation { 128 | currentStatus.SetStatusCondition(v1alpha1.NewRollingCondition( 129 | fmt.Sprintf("Waiting for deployment to be available: %d/%d available, %d/%d ready, %d/%d updated, %d/%d generation.", 130 | cachedDeployment.Status.AvailableReplicas, config.Replicas, 131 | cachedDeployment.Status.ReadyReplicas, config.Replicas, 132 | cachedDeployment.Status.UpdatedReplicas, config.Replicas, 133 | cachedDeployment.Status.ObservedGeneration, cachedDeployment.Generation, 134 | ))) 135 | if err := m.patchStatus(ctx, currentStatus); err != nil { 136 | QueueOps.RequeueAPIErr(ctx, err) 137 | return 138 | } 139 | QueueOps.RequeueAfter(ctx, 2*time.Second) 140 | return 141 | } 142 | 143 | // deployment is finished rolling out, remove condition 144 | if currentStatus.IsStatusConditionTrue(v1alpha1.ConditionTypeRolling) || 145 | currentStatus.IsStatusConditionTrue(v1alpha1.ConditionTypeRolloutError) { 146 | currentStatus.RemoveStatusCondition(v1alpha1.ConditionTypeRolling) 147 | currentStatus.RemoveStatusCondition(v1alpha1.ConditionTypeRolloutError) 148 | if err := m.patchStatus(ctx, currentStatus); err != nil { 149 | QueueOps.RequeueAPIErr(ctx, err) 150 | return 151 | } 152 | } 153 | 154 | m.next.Handle(ctx) 155 | } 156 | -------------------------------------------------------------------------------- /pkg/controller/pause.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/authzed/controller-idioms/handler" 7 | "github.com/authzed/controller-idioms/pause" 8 | 9 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 10 | "github.com/authzed/spicedb-operator/pkg/metadata" 11 | ) 12 | 13 | func NewPauseHandler( 14 | patchStatus func(ctx context.Context, patch *v1alpha1.SpiceDBCluster) error, 15 | next handler.ContextHandler, 16 | ) handler.Handler { 17 | return handler.NewHandler(pause.NewPauseContextHandler( 18 | QueueOps.Key, 19 | metadata.PausedControllerSelectorKey, 20 | CtxCluster, 21 | patchStatus, 22 | next, 23 | ), "pauseCluster") 24 | } 25 | -------------------------------------------------------------------------------- /pkg/controller/pause_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/authzed/controller-idioms/pause" 9 | "github.com/stretchr/testify/require" 10 | "golang.org/x/exp/slices" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/client-go/tools/record" 13 | 14 | "github.com/authzed/controller-idioms/handler" 15 | "github.com/authzed/controller-idioms/queue/fake" 16 | 17 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 18 | "github.com/authzed/spicedb-operator/pkg/metadata" 19 | ) 20 | 21 | func TestPauseHandler(t *testing.T) { 22 | var nextKey handler.Key = "next" 23 | tests := []struct { 24 | name string 25 | 26 | cluster *v1alpha1.SpiceDBCluster 27 | patchError error 28 | 29 | expectNext handler.Key 30 | expectEvents []string 31 | expectPatchStatus bool 32 | expectConditions []metav1.Condition 33 | expectRequeue bool 34 | expectDone bool 35 | }{ 36 | { 37 | name: "pauses when label found", 38 | cluster: &v1alpha1.SpiceDBCluster{ObjectMeta: metav1.ObjectMeta{ 39 | Labels: map[string]string{ 40 | metadata.PausedControllerSelectorKey: "", 41 | }, 42 | }}, 43 | expectPatchStatus: true, 44 | expectConditions: []metav1.Condition{pause.NewPausedCondition(metadata.PausedControllerSelectorKey)}, 45 | expectDone: true, 46 | }, 47 | { 48 | name: "requeues on pause patch error", 49 | cluster: &v1alpha1.SpiceDBCluster{ObjectMeta: metav1.ObjectMeta{ 50 | Labels: map[string]string{ 51 | metadata.PausedControllerSelectorKey: "", 52 | }, 53 | }}, 54 | patchError: fmt.Errorf("error patching"), 55 | expectPatchStatus: true, 56 | expectRequeue: true, 57 | }, 58 | { 59 | name: "no-op when label found and status is already paused", 60 | cluster: &v1alpha1.SpiceDBCluster{ 61 | ObjectMeta: metav1.ObjectMeta{ 62 | Labels: map[string]string{ 63 | metadata.PausedControllerSelectorKey: "", 64 | }, 65 | }, 66 | Status: v1alpha1.ClusterStatus{Conditions: []metav1.Condition{pause.NewPausedCondition(metadata.PausedControllerSelectorKey)}}, 67 | }, 68 | expectDone: true, 69 | }, 70 | { 71 | name: "removes condition when label is removed", 72 | cluster: &v1alpha1.SpiceDBCluster{Status: v1alpha1.ClusterStatus{Conditions: []metav1.Condition{ 73 | pause.NewPausedCondition(metadata.PausedControllerSelectorKey), 74 | }}}, 75 | expectPatchStatus: true, 76 | expectConditions: []metav1.Condition{}, 77 | expectNext: nextKey, 78 | }, 79 | { 80 | name: "requeues on unpause patch error", 81 | cluster: &v1alpha1.SpiceDBCluster{Status: v1alpha1.ClusterStatus{Conditions: []metav1.Condition{ 82 | pause.NewPausedCondition(metadata.PausedControllerSelectorKey), 83 | }}}, 84 | patchError: fmt.Errorf("error patching"), 85 | expectPatchStatus: true, 86 | expectRequeue: true, 87 | }, 88 | { 89 | name: "unpauses a self-paused object", 90 | cluster: &v1alpha1.SpiceDBCluster{Status: v1alpha1.ClusterStatus{Conditions: []metav1.Condition{ 91 | pause.NewSelfPausedCondition(metadata.PausedControllerSelectorKey), 92 | }}}, 93 | expectPatchStatus: true, 94 | expectConditions: []metav1.Condition{}, 95 | expectNext: nextKey, 96 | }, 97 | } 98 | for _, tt := range tests { 99 | t.Run(tt.name, func(t *testing.T) { 100 | ctrls := &fake.FakeInterface{} 101 | recorder := record.NewFakeRecorder(1) 102 | patchCalled := false 103 | 104 | ctx := context.Background() 105 | ctx = QueueOps.WithValue(ctx, ctrls) 106 | ctx = CtxClusterNN.WithValue(ctx, tt.cluster.NamespacedName()) 107 | ctx = CtxCluster.WithValue(ctx, tt.cluster) 108 | ctx = CtxCluster.WithValue(ctx, tt.cluster) 109 | var called handler.Key 110 | NewPauseHandler(func(_ context.Context, patch *v1alpha1.SpiceDBCluster) error { 111 | patchCalled = true 112 | 113 | if tt.patchError != nil { 114 | return tt.patchError 115 | } 116 | 117 | require.Truef(t, slices.EqualFunc(tt.expectConditions, patch.Status.Conditions, func(a, b metav1.Condition) bool { 118 | return a.Type == b.Type && 119 | a.Status == b.Status && 120 | a.ObservedGeneration == b.ObservedGeneration && 121 | a.Message == b.Message && 122 | a.Reason == b.Reason 123 | }), "conditions not equal:\na: %#v\nb: %#v", tt.expectConditions, patch.Status.Conditions) 124 | 125 | return nil 126 | }, handler.ContextHandlerFunc(func(_ context.Context) { 127 | called = nextKey 128 | })).Handle(ctx) 129 | 130 | require.Equal(t, tt.expectPatchStatus, patchCalled) 131 | require.Equal(t, tt.expectNext, called) 132 | require.Equal(t, tt.expectRequeue, ctrls.RequeueAPIErrCallCount() == 1) 133 | require.Equal(t, tt.expectDone, ctrls.DoneCallCount() == 1) 134 | ExpectEvents(t, recorder, tt.expectEvents) 135 | }) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /pkg/controller/run_migration.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "crypto/subtle" 6 | "time" 7 | 8 | batchv1 "k8s.io/api/batch/v1" 9 | "k8s.io/apimachinery/pkg/types" 10 | applybatchv1 "k8s.io/client-go/applyconfigurations/batch/v1" 11 | 12 | "github.com/authzed/controller-idioms/handler" 13 | 14 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 15 | "github.com/authzed/spicedb-operator/pkg/metadata" 16 | ) 17 | 18 | type MigrationRunHandler struct { 19 | patchStatus func(ctx context.Context, patch *v1alpha1.SpiceDBCluster) error 20 | applyJob func(ctx context.Context, job *applybatchv1.JobApplyConfiguration) error 21 | deleteJob func(ctx context.Context, nn types.NamespacedName) error 22 | next handler.ContextHandler 23 | } 24 | 25 | func (m *MigrationRunHandler) Handle(ctx context.Context) { 26 | // TODO: setting status is unconditional, should happen in a separate handler 27 | currentStatus := CtxCluster.MustValue(ctx) 28 | config := CtxConfig.MustValue(ctx) 29 | currentStatus.SetStatusCondition(v1alpha1.NewMigratingCondition(config.DatastoreEngine, config.TargetMigration)) 30 | if err := m.patchStatus(ctx, currentStatus); err != nil { 31 | QueueOps.RequeueErr(ctx, err) 32 | return 33 | } 34 | ctx = CtxCluster.WithValue(ctx, currentStatus) 35 | 36 | jobs := CtxJobs.MustValue(ctx) 37 | migrationHash := CtxMigrationHash.Value(ctx) 38 | 39 | matchingObjs := make([]*batchv1.Job, 0) 40 | extraObjs := make([]*batchv1.Job, 0) 41 | for _, o := range jobs { 42 | annotations := o.GetAnnotations() 43 | if annotations == nil { 44 | extraObjs = append(extraObjs, o) 45 | } 46 | if subtle.ConstantTimeCompare([]byte(annotations[metadata.SpiceDBMigrationRequirementsKey]), []byte(migrationHash)) == 1 { 47 | matchingObjs = append(matchingObjs, o) 48 | } else { 49 | extraObjs = append(extraObjs, o) 50 | } 51 | } 52 | 53 | if len(matchingObjs) == 0 { 54 | // apply if no matching object in controller 55 | err := m.applyJob(ctx, CtxConfig.MustValue(ctx).MigrationJob(migrationHash)) 56 | if err != nil { 57 | QueueOps.RequeueAPIErr(ctx, err) 58 | return 59 | } 60 | } 61 | 62 | // delete extra objects 63 | for _, o := range extraObjs { 64 | if err := m.deleteJob(ctx, types.NamespacedName{ 65 | Namespace: o.GetNamespace(), 66 | Name: o.GetName(), 67 | }); err != nil { 68 | QueueOps.RequeueAPIErr(ctx, err) 69 | return 70 | } 71 | } 72 | 73 | // job with correct hash exists 74 | if len(matchingObjs) >= 1 { 75 | ctx = CtxCurrentMigrationJob.WithValue(ctx, matchingObjs[0]) 76 | m.next.Handle(ctx) 77 | return 78 | } 79 | 80 | // if we had to create a job, requeue after a wait since the job takes time 81 | QueueOps.RequeueAfter(ctx, 5*time.Second) 82 | } 83 | -------------------------------------------------------------------------------- /pkg/controller/run_migration_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/require" 9 | batchv1 "k8s.io/api/batch/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/types" 12 | applybatchv1 "k8s.io/client-go/applyconfigurations/batch/v1" 13 | 14 | "github.com/authzed/controller-idioms/handler" 15 | "github.com/authzed/controller-idioms/queue/fake" 16 | 17 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 18 | "github.com/authzed/spicedb-operator/pkg/config" 19 | "github.com/authzed/spicedb-operator/pkg/metadata" 20 | ) 21 | 22 | func TestRunMigrationHandler(t *testing.T) { 23 | testHash := "hashhashhashhashhash" 24 | matchingJob := &batchv1.Job{ 25 | ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 26 | metadata.SpiceDBMigrationRequirementsKey: testHash, 27 | }}, 28 | } 29 | tests := []struct { 30 | name string 31 | 32 | clusterStatus *v1alpha1.SpiceDBCluster 33 | config config.Config 34 | existingJobs []*batchv1.Job 35 | migrationHash string 36 | jobApplyErr error 37 | jobDeleteErr error 38 | 39 | expectApply bool 40 | expectDelete bool 41 | expectNext bool 42 | expectCtxCluster *v1alpha1.ClusterStatus 43 | expectCtxCurrentMigrationJob *batchv1.Job 44 | expectRequeueErr error 45 | expectRequeue bool 46 | expectRequeueAfter time.Duration 47 | expectDone bool 48 | }{ 49 | { 50 | name: "creates if no matching job", 51 | clusterStatus: &v1alpha1.SpiceDBCluster{Status: v1alpha1.ClusterStatus{}}, 52 | existingJobs: []*batchv1.Job{}, 53 | migrationHash: testHash, 54 | expectApply: true, 55 | expectRequeueAfter: 5 * time.Second, 56 | }, 57 | { 58 | name: "no-ops if exactly 1 matching job", 59 | clusterStatus: &v1alpha1.SpiceDBCluster{Status: v1alpha1.ClusterStatus{}}, 60 | existingJobs: []*batchv1.Job{matchingJob}, 61 | migrationHash: testHash, 62 | expectNext: true, 63 | }, 64 | { 65 | name: "deletes non-matching job and creates matching job", 66 | clusterStatus: &v1alpha1.SpiceDBCluster{Status: v1alpha1.ClusterStatus{}}, 67 | existingJobs: []*batchv1.Job{{ 68 | ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 69 | metadata.SpiceDBMigrationRequirementsKey: "nope", 70 | }}, 71 | }}, 72 | migrationHash: testHash, 73 | expectDelete: true, 74 | expectApply: true, 75 | }, 76 | { 77 | name: "deletes non-matching job and leaves existing matching job", 78 | clusterStatus: &v1alpha1.SpiceDBCluster{Status: v1alpha1.ClusterStatus{}}, 79 | existingJobs: []*batchv1.Job{matchingJob, { 80 | ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ 81 | metadata.SpiceDBMigrationRequirementsKey: "nope", 82 | }}, 83 | }}, 84 | migrationHash: testHash, 85 | expectDelete: true, 86 | expectNext: true, 87 | }, 88 | } 89 | for _, tt := range tests { 90 | t.Run(tt.name, func(t *testing.T) { 91 | tt := tt 92 | ctrls := &fake.FakeInterface{} 93 | applyCalled := false 94 | deleteCalled := false 95 | nextCalled := false 96 | 97 | ctx := CtxCluster.WithValue(context.Background(), tt.clusterStatus) 98 | ctx = QueueOps.WithValue(ctx, ctrls) 99 | ctx = CtxConfig.WithValue(ctx, &tt.config) 100 | ctx = CtxJobs.WithBox(ctx) 101 | ctx = CtxJobs.WithValue(ctx, tt.existingJobs) 102 | ctx = CtxMigrationHash.WithValue(ctx, tt.migrationHash) 103 | 104 | h := &MigrationRunHandler{ 105 | patchStatus: func(_ context.Context, _ *v1alpha1.SpiceDBCluster) error { 106 | return nil 107 | }, 108 | applyJob: func(_ context.Context, _ *applybatchv1.JobApplyConfiguration) error { 109 | applyCalled = true 110 | return tt.jobApplyErr 111 | }, 112 | deleteJob: func(_ context.Context, _ types.NamespacedName) error { 113 | deleteCalled = true 114 | return tt.jobDeleteErr 115 | }, 116 | next: handler.ContextHandlerFunc(func(ctx context.Context) { 117 | nextCalled = true 118 | require.Equal(t, matchingJob, CtxCurrentMigrationJob.MustValue(ctx)) 119 | }), 120 | } 121 | h.Handle(ctx) 122 | 123 | require.True(t, CtxCluster.MustValue(ctx).IsStatusConditionTrue(v1alpha1.ConditionTypeMigrating)) 124 | require.Equal(t, tt.expectApply, applyCalled) 125 | require.Equal(t, tt.expectDelete, deleteCalled) 126 | require.Equal(t, tt.expectNext, nextCalled) 127 | if tt.expectRequeueErr != nil { 128 | require.Equal(t, 1, ctrls.RequeueErrCallCount()) 129 | require.Equal(t, tt.expectRequeueErr, ctrls.RequeueErrArgsForCall(0)) 130 | } 131 | require.Equal(t, tt.expectRequeue, ctrls.RequeueCallCount() == 1) 132 | if tt.expectRequeueAfter != 0 { 133 | require.Equal(t, 1, ctrls.RequeueAfterCallCount()) 134 | require.Equal(t, tt.expectRequeueAfter, ctrls.RequeueAfterArgsForCall(0)) 135 | } 136 | }) 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /pkg/controller/secret_adoption.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/apimachinery/pkg/types" 8 | applycorev1 "k8s.io/client-go/applyconfigurations/core/v1" 9 | "k8s.io/client-go/tools/record" 10 | 11 | "github.com/authzed/controller-idioms/adopt" 12 | "github.com/authzed/controller-idioms/handler" 13 | "github.com/authzed/controller-idioms/typed" 14 | 15 | "github.com/authzed/spicedb-operator/pkg/metadata" 16 | ) 17 | 18 | const EventSecretAdoptedBySpiceDBCluster = "SecretAdoptedBySpiceDB" 19 | 20 | func NewSecretAdoptionHandler(recorder record.EventRecorder, getFromCache func(ctx context.Context) (*corev1.Secret, error), missingFunc func(ctx context.Context, err error), secretIndexer *typed.Indexer[*corev1.Secret], secretApplyFunc adopt.ApplyFunc[*corev1.Secret, *applycorev1.SecretApplyConfiguration], existsFunc func(ctx context.Context, name types.NamespacedName) error, next handler.Handler) handler.Handler { 21 | return handler.NewHandler(&adopt.AdoptionHandler[*corev1.Secret, *applycorev1.SecretApplyConfiguration]{ 22 | OperationsContext: QueueOps, 23 | ControllerFieldManager: metadata.FieldManager, 24 | AdopteeCtx: CtxSecretNN, 25 | OwnerCtx: CtxClusterNN, 26 | AdoptedCtx: CtxSecret, 27 | ObjectAdoptedFunc: func(ctx context.Context, secret *corev1.Secret) { 28 | recorder.Eventf(secret, corev1.EventTypeNormal, EventSecretAdoptedBySpiceDBCluster, "Secret was referenced as the secret source for SpiceDBCluster %s; it has been labelled to mark it as part of the configuration for that controller.", CtxClusterNN.MustValue(ctx).String()) 29 | }, 30 | ObjectMissingFunc: missingFunc, 31 | GetFromCache: getFromCache, 32 | Indexer: secretIndexer, 33 | IndexName: metadata.OwningClusterIndex, 34 | Labels: map[string]string{metadata.OperatorManagedLabelKey: metadata.OperatorManagedLabelValue}, 35 | NewPatch: func(nn types.NamespacedName) *applycorev1.SecretApplyConfiguration { 36 | return applycorev1.Secret(nn.Name, nn.Namespace) 37 | }, 38 | OwnerAnnotationPrefix: metadata.OwnerAnnotationKeyPrefix, 39 | OwnerAnnotationKeyFunc: func(owner types.NamespacedName) string { 40 | return metadata.OwnerAnnotationKeyPrefix + owner.Name 41 | }, 42 | OwnerFieldManagerFunc: func(owner types.NamespacedName) string { 43 | return "spicedbcluster-owner-" + owner.Namespace + "-" + owner.Name 44 | }, 45 | ApplyFunc: secretApplyFunc, 46 | ExistsFunc: existsFunc, 47 | Next: next, 48 | }, "adoptSecret") 49 | } 50 | -------------------------------------------------------------------------------- /pkg/controller/self_pause.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/authzed/controller-idioms/handler" 7 | "github.com/authzed/controller-idioms/pause" 8 | 9 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 10 | "github.com/authzed/spicedb-operator/pkg/metadata" 11 | ) 12 | 13 | const HandlerSelfPauseKey handler.Key = "nextSelfPause" 14 | 15 | func NewSelfPauseHandler( 16 | patch func(ctx context.Context, patch *v1alpha1.SpiceDBCluster) error, 17 | patchStatus func(ctx context.Context, patch *v1alpha1.SpiceDBCluster) error, 18 | ) handler.Handler { 19 | return handler.NewHandler(pause.NewSelfPauseHandler( 20 | QueueOps.Key, 21 | metadata.PausedControllerSelectorKey, 22 | CtxSelfPauseObject, 23 | patch, 24 | patchStatus, 25 | ), HandlerSelfPauseKey) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/controller/validate_config.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/apimachinery/pkg/api/meta" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/client-go/tools/record" 10 | "k8s.io/kubectl/pkg/util/openapi" 11 | 12 | "github.com/authzed/controller-idioms/handler" 13 | "github.com/authzed/controller-idioms/hash" 14 | 15 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 16 | "github.com/authzed/spicedb-operator/pkg/config" 17 | ) 18 | 19 | const EventInvalidSpiceDBConfig = "InvalidSpiceDBConfig" 20 | 21 | type ValidateConfigHandler struct { 22 | recorder record.EventRecorder 23 | resources openapi.Resources 24 | patchStatus func(ctx context.Context, patch *v1alpha1.SpiceDBCluster) error 25 | next handler.ContextHandler 26 | } 27 | 28 | func (c *ValidateConfigHandler) Handle(ctx context.Context) { 29 | cluster := CtxCluster.MustValue(ctx) 30 | secret := CtxSecret.Value(ctx) 31 | operatorConfig := CtxOperatorConfig.MustValue(ctx) 32 | 33 | validatedConfig, warning, err := config.NewConfig(cluster, operatorConfig, secret, c.resources) 34 | if err != nil { 35 | failedCondition := v1alpha1.NewInvalidConfigCondition(CtxSecretHash.Value(ctx), err) 36 | if existing := cluster.FindStatusCondition(v1alpha1.ConditionValidatingFailed); existing != nil && existing.Message == failedCondition.Message { 37 | QueueOps.Done(ctx) 38 | return 39 | } 40 | cluster.Status.ObservedGeneration = cluster.GetGeneration() 41 | cluster.RemoveStatusCondition(v1alpha1.ConditionTypeValidating) 42 | cluster.SetStatusCondition(failedCondition) 43 | if err := c.patchStatus(ctx, cluster); err != nil { 44 | QueueOps.RequeueAPIErr(ctx, err) 45 | return 46 | } 47 | c.recorder.Eventf(cluster, corev1.EventTypeWarning, EventInvalidSpiceDBConfig, "invalid config: %v", err) 48 | // if the config is invalid, there's no work to do until it has changed 49 | QueueOps.Done(ctx) 50 | return 51 | } 52 | 53 | var warningCondition *metav1.Condition 54 | if warning != nil { 55 | cond := v1alpha1.NewConfigWarningCondition(warning) 56 | warningCondition = &cond 57 | } 58 | 59 | migrationHash := hash.SecureObject(validatedConfig.MigrationConfig) 60 | ctx = CtxMigrationHash.WithValue(ctx, migrationHash) 61 | 62 | computedStatus := v1alpha1.ClusterStatus{ 63 | ObservedGeneration: cluster.GetGeneration(), 64 | TargetMigrationHash: migrationHash, 65 | CurrentMigrationHash: cluster.Status.CurrentMigrationHash, 66 | SecretHash: cluster.Status.SecretHash, 67 | Image: validatedConfig.TargetSpiceDBImage, 68 | Migration: validatedConfig.TargetMigration, 69 | Phase: validatedConfig.TargetPhase, 70 | CurrentVersion: validatedConfig.SpiceDBVersion, 71 | Conditions: *cluster.GetStatusConditions(), 72 | } 73 | if version := validatedConfig.SpiceDBVersion; version != nil { 74 | computedStatus.AvailableVersions, err = operatorConfig.UpdateGraph.AvailableVersions(validatedConfig.DatastoreEngine, *version) 75 | if err != nil { 76 | QueueOps.RequeueErr(ctx, err) 77 | return 78 | } 79 | } 80 | meta.RemoveStatusCondition(&computedStatus.Conditions, v1alpha1.ConditionValidatingFailed) 81 | meta.RemoveStatusCondition(&computedStatus.Conditions, v1alpha1.ConditionTypeValidating) 82 | if warningCondition != nil { 83 | meta.SetStatusCondition(&computedStatus.Conditions, *warningCondition) 84 | } else { 85 | meta.RemoveStatusCondition(&computedStatus.Conditions, v1alpha1.ConditionTypeConfigWarnings) 86 | } 87 | 88 | // Remove invalid config status and set image and hash 89 | if !cluster.Status.Equals(computedStatus) { 90 | cluster.Status = computedStatus 91 | if err := c.patchStatus(ctx, cluster); err != nil { 92 | QueueOps.RequeueAPIErr(ctx, err) 93 | return 94 | } 95 | } 96 | 97 | ctx = CtxConfig.WithValue(ctx, validatedConfig) 98 | ctx = CtxCluster.WithValue(ctx, cluster) 99 | c.next.Handle(ctx) 100 | } 101 | -------------------------------------------------------------------------------- /pkg/controller/wait_for_migrations.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | batchv1 "k8s.io/api/batch/v1" 9 | corev1 "k8s.io/api/core/v1" 10 | "k8s.io/apimachinery/pkg/util/runtime" 11 | "k8s.io/client-go/tools/record" 12 | 13 | "github.com/authzed/controller-idioms/handler" 14 | 15 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 16 | ) 17 | 18 | const EventMigrationsComplete = "MigrationsCompleted" 19 | 20 | type WaitForMigrationsHandler struct { 21 | recorder record.EventRecorder 22 | nextSelfPause handler.ContextHandler 23 | nextDeploymentHandler handler.ContextHandler 24 | } 25 | 26 | func (m *WaitForMigrationsHandler) Handle(ctx context.Context) { 27 | job := CtxCurrentMigrationJob.MustValue(ctx) 28 | 29 | // if migration failed entirely, pause so we can diagnose 30 | if c := findJobCondition(job, batchv1.JobFailed); c != nil && c.Status == corev1.ConditionTrue { 31 | currentStatus := CtxCluster.MustValue(ctx) 32 | config := CtxConfig.MustValue(ctx) 33 | err := fmt.Errorf("migration job failed: %s", c.Message) 34 | runtime.HandleError(err) 35 | currentStatus.SetStatusCondition(v1alpha1.NewMigrationFailedCondition(config.DatastoreEngine, "head", err)) 36 | ctx = CtxSelfPauseObject.WithValue(ctx, currentStatus) 37 | m.nextSelfPause.Handle(ctx) 38 | return 39 | } 40 | 41 | // if done, go to the nextDeploymentHandler step 42 | if jobConditionHasStatus(job, batchv1.JobComplete, corev1.ConditionTrue) { 43 | m.recorder.Eventf(CtxCluster.MustValue(ctx), corev1.EventTypeNormal, EventMigrationsComplete, "Migrations completed for %s", CtxConfig.MustValue(ctx).TargetSpiceDBImage) 44 | m.nextDeploymentHandler.Handle(ctx) 45 | return 46 | } 47 | 48 | // otherwise, it's created but still running, just wait 49 | QueueOps.RequeueAfter(ctx, 5*time.Second) 50 | } 51 | 52 | func findJobCondition(job *batchv1.Job, conditionType batchv1.JobConditionType) *batchv1.JobCondition { 53 | if job == nil { 54 | return nil 55 | } 56 | conditions := job.Status.Conditions 57 | for i := range conditions { 58 | if conditions[i].Type == conditionType { 59 | return &conditions[i] 60 | } 61 | } 62 | return nil 63 | } 64 | 65 | func jobConditionHasStatus(job *batchv1.Job, conditionType batchv1.JobConditionType, status corev1.ConditionStatus) bool { 66 | c := findJobCondition(job, conditionType) 67 | if c == nil { 68 | return false 69 | } 70 | return c.Status == status 71 | } 72 | -------------------------------------------------------------------------------- /pkg/controller/wait_for_migrations_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/require" 9 | batchv1 "k8s.io/api/batch/v1" 10 | corev1 "k8s.io/api/core/v1" 11 | "k8s.io/client-go/tools/record" 12 | 13 | "github.com/authzed/controller-idioms/handler" 14 | "github.com/authzed/controller-idioms/queue/fake" 15 | 16 | "github.com/authzed/spicedb-operator/pkg/apis/authzed/v1alpha1" 17 | "github.com/authzed/spicedb-operator/pkg/config" 18 | ) 19 | 20 | func TestWaitForMigrationsHandler(t *testing.T) { 21 | tests := []struct { 22 | name string 23 | 24 | migrationJob *batchv1.Job 25 | 26 | expectNext handler.Key 27 | expectRequeueAfter time.Duration 28 | expectEvents []string 29 | }{ 30 | { 31 | name: "job is still running, requeue with delay", 32 | migrationJob: &batchv1.Job{}, 33 | expectRequeueAfter: 5 * time.Second, 34 | }, 35 | { 36 | name: "job is done, check deployments", 37 | migrationJob: &batchv1.Job{Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{ 38 | Type: batchv1.JobComplete, 39 | Status: corev1.ConditionTrue, 40 | }}}}, 41 | expectEvents: []string{ 42 | "Normal MigrationsCompleted Migrations completed for test", 43 | }, 44 | expectNext: HandlerDeploymentKey, 45 | }, 46 | { 47 | name: "job failed, pause reconciliation", 48 | migrationJob: &batchv1.Job{Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{ 49 | Type: batchv1.JobFailed, 50 | Status: corev1.ConditionTrue, 51 | }}}}, 52 | expectNext: HandlerSelfPauseKey, 53 | }, 54 | } 55 | for _, tt := range tests { 56 | t.Run(tt.name, func(t *testing.T) { 57 | ctrls := &fake.FakeInterface{} 58 | 59 | ctx := CtxConfig.WithValue(context.Background(), &config.Config{MigrationConfig: config.MigrationConfig{TargetSpiceDBImage: "test"}}) 60 | ctx = QueueOps.WithValue(ctx, ctrls) 61 | ctx = CtxCluster.WithValue(ctx, &v1alpha1.SpiceDBCluster{}) 62 | ctx = CtxCurrentMigrationJob.WithValue(ctx, tt.migrationJob) 63 | 64 | recorder := record.NewFakeRecorder(1) 65 | 66 | var called handler.Key 67 | h := &WaitForMigrationsHandler{ 68 | recorder: recorder, 69 | nextSelfPause: handler.ContextHandlerFunc(func(_ context.Context) { 70 | called = HandlerSelfPauseKey 71 | }), 72 | nextDeploymentHandler: handler.ContextHandlerFunc(func(_ context.Context) { 73 | called = HandlerDeploymentKey 74 | }), 75 | } 76 | h.Handle(ctx) 77 | 78 | require.Equal(t, tt.expectNext, called) 79 | ExpectEvents(t, recorder, tt.expectEvents) 80 | 81 | if tt.expectRequeueAfter != 0 { 82 | require.Equal(t, 1, ctrls.RequeueAfterCallCount()) 83 | require.Equal(t, tt.expectRequeueAfter, ctrls.RequeueAfterArgsForCall(0)) 84 | } 85 | }) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /pkg/crds/crds.go: -------------------------------------------------------------------------------- 1 | package crds 2 | 3 | import ( 4 | "context" 5 | "embed" 6 | 7 | "k8s.io/client-go/rest" 8 | 9 | libbootstrap "github.com/authzed/controller-idioms/bootstrap" 10 | ) 11 | 12 | //go:embed *.yaml 13 | var crdFS embed.FS 14 | 15 | func BootstrapCRD(ctx context.Context, restConfig *rest.Config) error { 16 | return libbootstrap.CRDs(ctx, restConfig, crdFS, ".") 17 | } 18 | -------------------------------------------------------------------------------- /pkg/metadata/keys.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/authzed/controller-idioms/adopt" 8 | "k8s.io/apimachinery/pkg/api/meta" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/apimachinery/pkg/labels" 11 | "k8s.io/apimachinery/pkg/runtime" 12 | "k8s.io/apimachinery/pkg/runtime/schema" 13 | "k8s.io/apimachinery/pkg/types" 14 | "k8s.io/client-go/tools/cache" 15 | "k8s.io/utils/ptr" 16 | ) 17 | 18 | const ( 19 | OwningClusterIndex = "owning-cluster" 20 | OperatorManagedLabelKey = "authzed.com/managed-by" 21 | OperatorManagedLabelValue = "operator" 22 | OwnerLabelKey = "authzed.com/cluster" 23 | OwnerAnnotationKeyPrefix = "authzed.com.cluster-owner/" 24 | ComponentLabelKey = "authzed.com/cluster-component" 25 | ComponentSpiceDBLabelValue = "spicedb" 26 | ComponentMigrationJobLabelValue = "migration-job" 27 | ComponentServiceAccountLabel = "spicedb-serviceaccount" 28 | ComponentRoleLabel = "spicedb-role" 29 | ComponentServiceLabel = "spicedb-service" 30 | ComponentRoleBindingLabel = "spicedb-rolebinding" 31 | SpiceDBMigrationRequirementsKey = "authzed.com/spicedb-migration" 32 | SpiceDBTargetMigrationKey = "authzed.com/spicedb-target-migration" 33 | SpiceDBSecretRequirementsKey = "authzed.com/spicedb-secret" // nolint: gosec 34 | SpiceDBConfigKey = "authzed.com/spicedb-configuration" 35 | FieldManager = "spicedb-operator" 36 | ) 37 | 38 | var ( 39 | ApplyForceOwned = metav1.ApplyOptions{FieldManager: FieldManager, Force: true} 40 | PatchForceOwned = metav1.PatchOptions{FieldManager: FieldManager, Force: ptr.To(true)} 41 | ManagedDependentSelector = MustParseSelector(fmt.Sprintf("%s=%s", OperatorManagedLabelKey, OperatorManagedLabelValue)) 42 | ) 43 | 44 | func SelectorForComponent(owner, component string) labels.Selector { 45 | return labels.SelectorFromSet(LabelsForComponent(owner, component)) 46 | } 47 | 48 | func LabelsForComponent(owner, component string) map[string]string { 49 | return map[string]string{ 50 | OwnerLabelKey: owner, 51 | ComponentLabelKey: component, 52 | OperatorManagedLabelKey: OperatorManagedLabelValue, 53 | } 54 | } 55 | 56 | func GVRMetaNamespaceKeyer(gvr schema.GroupVersionResource, key string) string { 57 | return fmt.Sprintf("%s.%s.%s::%s", gvr.Resource, gvr.Version, gvr.Group, key) 58 | } 59 | 60 | func GVRMetaNamespaceKeyFunc(gvr schema.GroupVersionResource, obj interface{}) (string, error) { 61 | if d, ok := obj.(cache.DeletedFinalStateUnknown); ok { 62 | return d.Key, nil 63 | } 64 | key, err := cache.MetaNamespaceKeyFunc(obj) 65 | if err != nil { 66 | return "", err 67 | } 68 | return GVRMetaNamespaceKeyer(gvr, key), nil 69 | } 70 | 71 | func SplitGVRMetaNamespaceKey(key string) (gvr *schema.GroupVersionResource, namespace, name string, err error) { 72 | before, after, ok := strings.Cut(key, "::") 73 | if !ok { 74 | err = fmt.Errorf("error parsing key: %s", key) 75 | return 76 | } 77 | gvr, _ = schema.ParseResourceArg(before) 78 | if gvr == nil { 79 | err = fmt.Errorf("error parsing gvr from key: %s", before) 80 | return 81 | } 82 | namespace, name, err = cache.SplitMetaNamespaceKey(after) 83 | return 84 | } 85 | 86 | func GetClusterKeyFromMeta(in interface{}) ([]string, error) { 87 | obj := in.(runtime.Object) 88 | objMeta, err := meta.Accessor(obj) 89 | if err != nil { 90 | return nil, err 91 | } 92 | 93 | clusterNames, err := adopt.OwnerKeysFromMeta(OwnerAnnotationKeyPrefix)(obj) 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | objLabels := objMeta.GetLabels() 99 | if len(objLabels) > 0 { 100 | clusterName, ok := objLabels[OwnerLabelKey] 101 | if ok { 102 | nn := types.NamespacedName{Name: clusterName, Namespace: objMeta.GetNamespace()} 103 | clusterNames = append(clusterNames, nn.String()) 104 | } 105 | } 106 | 107 | return clusterNames, nil 108 | } 109 | -------------------------------------------------------------------------------- /pkg/metadata/pause.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/labels" 5 | ) 6 | 7 | const PausedControllerSelectorKey = "authzed.com/controller-paused" 8 | 9 | var NotPausedSelector = MustParseSelector("!" + PausedControllerSelectorKey) 10 | 11 | func MustParseSelector(selector string) labels.Selector { 12 | s, err := labels.Parse(selector) 13 | if err != nil { 14 | panic(err) 15 | } 16 | return s 17 | } 18 | -------------------------------------------------------------------------------- /pkg/updates/memory.go: -------------------------------------------------------------------------------- 1 | package updates 2 | 3 | import ( 4 | "fmt" 5 | 6 | "golang.org/x/exp/maps" 7 | ) 8 | 9 | // EdgeSet maps a node id to a list of node ids that it can update to 10 | type EdgeSet map[string][]string 11 | 12 | // NodeSet maps a node id to an index in the OrderedNodes array 13 | type NodeSet map[string]int 14 | 15 | // MemorySource is an in-memory implementation of Source. 16 | // It's an oracle to answer update questions for an installed currentVersion. 17 | type MemorySource struct { 18 | // OrderedNodes is an ordered list of all nodes. Lower index == newer currentVersion. 19 | OrderedNodes []State 20 | // Nodes is a helper to lookup a node by id 21 | Nodes NodeSet 22 | // Edges contains the edgeset for this source. 23 | Edges EdgeSet 24 | } 25 | 26 | var _ Source = (*MemorySource)(nil) 27 | 28 | func (m *MemorySource) NextVersion(from string) string { 29 | if edges, ok := m.Edges[from]; ok && len(edges) > 0 { 30 | return edges[len(edges)-1] 31 | } 32 | return "" 33 | } 34 | 35 | func (m *MemorySource) NextVersionWithoutMigrations(from string) (found string) { 36 | initial := m.OrderedNodes[m.Nodes[from]] 37 | if to, ok := m.Edges[from]; ok && len(to) > 0 { 38 | for _, n := range m.Edges[from] { 39 | node := m.OrderedNodes[m.Nodes[n]] 40 | 41 | // if the phase and migration match the current node, no migrations 42 | // are required 43 | if initial.Phase == node.Phase && initial.Migration == node.Migration { 44 | found = n 45 | } else { 46 | break 47 | } 48 | } 49 | } 50 | return found 51 | } 52 | 53 | func (m *MemorySource) LatestVersion(id string) string { 54 | if len(m.OrderedNodes) == 0 || id == m.OrderedNodes[0].ID { 55 | return "" 56 | } 57 | return m.OrderedNodes[0].ID 58 | } 59 | 60 | func (m *MemorySource) State(id string) State { 61 | index, ok := m.Nodes[id] 62 | if !ok { 63 | return State{} 64 | } 65 | return m.OrderedNodes[index] 66 | } 67 | 68 | func (m *MemorySource) Subgraph(head string) (Source, error) { 69 | // copy the ordered node list from `to` onward 70 | var index int 71 | if len(head) > 0 { 72 | index = m.Nodes[head] 73 | } 74 | orderedNodes := make([]State, len(m.OrderedNodes)-index) 75 | copy(orderedNodes, m.OrderedNodes[index:len(m.OrderedNodes)]) 76 | 77 | nodeSet := make(map[string]int, len(orderedNodes)) 78 | for i, n := range orderedNodes { 79 | nodeSet[n.ID] = i 80 | } 81 | 82 | edges := make(map[string][]string) 83 | for from, to := range m.Edges { 84 | // skip edges where from is not in the node set 85 | if _, ok := nodeSet[from]; !ok { 86 | continue 87 | } 88 | _, ok := edges[from] 89 | if !ok { 90 | edges[from] = make([]string, 0) 91 | } 92 | for _, n := range to { 93 | // skip edges where to is not in the node set 94 | if _, ok := nodeSet[n]; !ok { 95 | continue 96 | } 97 | edges[from] = append(edges[from], n) 98 | } 99 | } 100 | 101 | return &MemorySource{Nodes: nodeSet, Edges: edges, OrderedNodes: orderedNodes}, nil 102 | } 103 | 104 | func (m *MemorySource) validateAllNodesPathToHead() error { 105 | head := m.OrderedNodes[0].ID 106 | for _, n := range m.OrderedNodes { 107 | if n.ID == head { 108 | continue 109 | } 110 | visited := make(map[string]struct{}, 0) 111 | // chasing current should lead to head 112 | for current := m.NextVersion(n.ID); current != head; current = m.NextVersion(current) { 113 | if _, ok := visited[current]; ok { 114 | return fmt.Errorf("channel cycle detected: %v", append(maps.Keys(visited), current)) 115 | } 116 | if current == "" { 117 | return fmt.Errorf("there is no path from %s to %s", n.ID, m.OrderedNodes[0].ID) 118 | } 119 | visited[current] = struct{}{} 120 | } 121 | } 122 | return nil 123 | } 124 | 125 | func NewMemorySource(nodes []State, edges EdgeSet) (Source, error) { 126 | if len(nodes) == 0 { 127 | return nil, fmt.Errorf("missing nodes") 128 | } else if len(edges) == 0 { 129 | return nil, fmt.Errorf("missing edges") 130 | } 131 | 132 | nodeSet := make(map[string]int, len(nodes)) 133 | for i, n := range nodes { 134 | if _, ok := nodeSet[n.ID]; ok { 135 | return nil, fmt.Errorf("more than one node with ID %s", n.ID) 136 | } 137 | nodeSet[n.ID] = i 138 | } 139 | 140 | for from, toSet := range edges { 141 | // ensure all edges reference nodes 142 | if _, ok := nodeSet[from]; !ok { 143 | return nil, fmt.Errorf("node list is missing node %s", from) 144 | } 145 | for _, to := range toSet { 146 | if _, ok := nodeSet[to]; !ok { 147 | return nil, fmt.Errorf("node list is missing node %s", to) 148 | } 149 | } 150 | if len(toSet) != 0 { 151 | continue 152 | } 153 | 154 | // The only node with no updates should be the head of the channel 155 | // i.e. the first thing in the ordered node list 156 | if from != nodes[0].ID { 157 | return nil, fmt.Errorf("%s has no outgoing edges, but it is not the head of the channel", from) 158 | } 159 | } 160 | 161 | return newMemorySourceFromValidatedNodes(nodeSet, edges, nodes) 162 | } 163 | 164 | func newMemorySourceFromValidatedNodes(nodeSet map[string]int, edges map[string][]string, nodes []State) (Source, error) { 165 | source := &MemorySource{Nodes: nodeSet, Edges: edges, OrderedNodes: nodes} 166 | 167 | if err := source.validateAllNodesPathToHead(); err != nil { 168 | return nil, err 169 | } 170 | 171 | // TODO: validate that the adjacency lists are in the same order as the node list 172 | // so that we can always assume further down the list is newer 173 | return source, nil 174 | } 175 | -------------------------------------------------------------------------------- /pkg/updates/source.go: -------------------------------------------------------------------------------- 1 | package updates 2 | 3 | // Source models a single stream of updates for an installed currentVersion. 4 | type Source interface { 5 | // NextVersionWithoutMigrations returns the newest currentVersion that has an edge that 6 | // does not require any migrations. 7 | NextVersionWithoutMigrations(from string) string 8 | 9 | // NextVersion returns the newest currentVersion that has an edge. 10 | // This currentVersion might include migrations. 11 | NextVersion(from string) string 12 | 13 | // LatestVersion returns the newest currentVersion that has some path through the 14 | // graph. 15 | // 16 | // If no path exists, returns the empty string. 17 | // 18 | // If different from `NextVersion`, that means multiple steps are 19 | // required (i.e. a multi-phase migration, or a required stopping point 20 | // in a series of updates). 21 | LatestVersion(from string) string 22 | 23 | // State returns the information that is required to update to the provided 24 | // node. 25 | State(id string) State 26 | 27 | // Subgraph returns a new Source that is a subgraph of the current source, 28 | // but where `head` is set to the provided node. 29 | Subgraph(head string) (Source, error) 30 | } 31 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "runtime/debug" 7 | "strings" 8 | ) 9 | 10 | // Version is this program's version string 11 | var Version string 12 | 13 | // UsageVersion introspects the process debug data for Go modules to return a 14 | // version string. 15 | func UsageVersion(includeDeps bool) string { 16 | bi, ok := debug.ReadBuildInfo() 17 | if !ok { 18 | panic("failed to read BuildInfo because the program was compiled with Go " + runtime.Version()) 19 | } 20 | 21 | if Version == "" { 22 | // The version wasn't set by ldflags, so fallback to the Go module version. 23 | // Although, this value is pretty much guaranteed to just be "(devel)". 24 | Version = bi.Main.Version 25 | } 26 | 27 | if !includeDeps { 28 | if Version == "(devel)" { 29 | return "spicedb-operator development build (unknown exact version)" 30 | } 31 | return "spicedb-operator " + Version 32 | } 33 | 34 | var b strings.Builder 35 | fmt.Fprintf(&b, "%s %s", bi.Path, Version) 36 | for _, dep := range bi.Deps { 37 | fmt.Fprintf(&b, "\n\t%s %s", dep.Path, dep.Version) 38 | } 39 | return b.String() 40 | } 41 | -------------------------------------------------------------------------------- /tools/generate-update-graph/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/authzed/spicedb-operator/pkg/updates" 9 | ) 10 | 11 | func TestEdgesFromPatterns(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | patterns map[string]string 15 | releases []updates.State 16 | want map[string][]string 17 | }{ 18 | { 19 | name: "single edge", 20 | releases: []updates.State{ 21 | {ID: "v1.0.0"}, 22 | {ID: "v1.1.0"}, 23 | }, 24 | patterns: map[string]string{ 25 | "v1.0.0": "1.1.0", 26 | }, 27 | want: map[string][]string{ 28 | "v1.0.0": {"v1.1.0"}, 29 | }, 30 | }, 31 | { 32 | name: "open edge range", 33 | releases: []updates.State{ 34 | {ID: "v1.0.0"}, 35 | {ID: "v1.1.0"}, 36 | {ID: "v1.2.0"}, 37 | }, 38 | patterns: map[string]string{ 39 | "v1.0.0": ">1.0.0", 40 | }, 41 | want: map[string][]string{ 42 | "v1.0.0": {"v1.1.0", "v1.2.0"}, 43 | }, 44 | }, 45 | { 46 | name: "closed edge range", 47 | releases: []updates.State{ 48 | {ID: "v1.0.0"}, 49 | {ID: "v1.1.0"}, 50 | {ID: "v1.2.0"}, 51 | }, 52 | patterns: map[string]string{ 53 | "v1.0.0": ">1.0.0 <=1.1.0", 54 | }, 55 | want: map[string][]string{ 56 | "v1.0.0": {"v1.1.0"}, 57 | }, 58 | }, 59 | { 60 | name: "edge range with deprecated release in range", 61 | releases: []updates.State{ 62 | {ID: "v1.0.0"}, 63 | {ID: "v1.1.0", Deprecated: true}, 64 | {ID: "v1.2.0"}, 65 | }, 66 | patterns: map[string]string{ 67 | "v1.0.0": ">1.0.0", 68 | }, 69 | want: map[string][]string{ 70 | "v1.0.0": {"v1.2.0"}, 71 | }, 72 | }, 73 | { 74 | name: "edge range with omitted version", 75 | releases: []updates.State{ 76 | {ID: "v1.0.0"}, 77 | {ID: "v1.1.0"}, 78 | {ID: "v1.2.0"}, 79 | {ID: "v1.3.0"}, 80 | }, 81 | patterns: map[string]string{ 82 | "v1.0.0": ">1.0.0 !1.2.0", 83 | }, 84 | want: map[string][]string{ 85 | "v1.0.0": {"v1.1.0", "v1.3.0"}, 86 | }, 87 | }, 88 | } 89 | for _, tt := range tests { 90 | t.Run(tt.name, func(t *testing.T) { 91 | got := edgesFromPatterns(tt.patterns, tt.releases) 92 | for from, to := range tt.want { 93 | require.ElementsMatch(t, to, got[from]) 94 | } 95 | for from, to := range got { 96 | require.ElementsMatch(t, to, tt.want[from]) 97 | } 98 | }) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /tools/go.mod: -------------------------------------------------------------------------------- 1 | module tools 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/authzed/spicedb-operator v0.0.0-00010101000000-000000000000 9 | github.com/blang/semver/v4 v4.0.0 10 | github.com/stretchr/testify v1.10.0 11 | sigs.k8s.io/yaml v1.4.0 12 | ) 13 | 14 | require ( 15 | github.com/authzed/controller-idioms v0.11.0 // indirect 16 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 17 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 18 | github.com/emicklei/go-restful/v3 v3.12.1 // indirect 19 | github.com/evanphx/json-patch v5.9.11+incompatible // indirect 20 | github.com/fatih/camelcase v1.0.0 // indirect 21 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 22 | github.com/go-logr/logr v1.4.2 // indirect 23 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 24 | github.com/go-openapi/jsonreference v0.21.0 // indirect 25 | github.com/go-openapi/swag v0.23.0 // indirect 26 | github.com/gogo/protobuf v1.3.2 // indirect 27 | github.com/golang/protobuf v1.5.4 // indirect 28 | github.com/google/gnostic-models v0.6.8 // indirect 29 | github.com/google/go-cmp v0.6.0 // indirect 30 | github.com/google/gofuzz v1.2.0 // indirect 31 | github.com/google/uuid v1.6.0 // indirect 32 | github.com/josharian/intern v1.0.0 // indirect 33 | github.com/json-iterator/go v1.1.12 // indirect 34 | github.com/jzelinskie/stringz v0.0.3 // indirect 35 | github.com/mailru/easyjson v0.7.7 // indirect 36 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 37 | github.com/modern-go/reflect2 v1.0.2 // indirect 38 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 39 | github.com/pkg/errors v0.9.1 // indirect 40 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 41 | github.com/samber/lo v1.49.1 // indirect 42 | github.com/x448/float16 v0.8.4 // indirect 43 | golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect 44 | golang.org/x/net v0.34.0 // indirect 45 | golang.org/x/oauth2 v0.23.0 // indirect 46 | golang.org/x/sys v0.29.0 // indirect 47 | golang.org/x/term v0.28.0 // indirect 48 | golang.org/x/text v0.21.0 // indirect 49 | golang.org/x/time v0.7.0 // indirect 50 | google.golang.org/protobuf v1.36.1 // indirect 51 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 52 | gopkg.in/inf.v0 v0.9.1 // indirect 53 | gopkg.in/yaml.v3 v3.0.1 // indirect 54 | k8s.io/api v0.32.3 // indirect 55 | k8s.io/apimachinery v0.32.3 // indirect 56 | k8s.io/client-go v0.32.3 // indirect 57 | k8s.io/klog/v2 v2.130.1 // indirect 58 | k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect 59 | k8s.io/kubectl v0.32.3 // indirect 60 | k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect 61 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect 62 | sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect 63 | ) 64 | 65 | replace github.com/authzed/spicedb-operator => ../ 66 | --------------------------------------------------------------------------------