├── .changes ├── header.tpl.md ├── unreleased │ └── .gitkeep ├── v0.5.31.md ├── v0.5.32.md ├── v0.6.0.md ├── v0.6.1.md ├── v0.6.2.md ├── v0.6.3.md └── v0.6.4.md ├── .changie.yaml ├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── 01_BUG_REPORT.md │ ├── 02_FEATURE_REQUEST.md │ └── 03_CODEBASE_IMPROVEMENT.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── compatibility-tests.yaml │ ├── create-release-pr.yml │ ├── run-tests.yml │ └── upload-artifacts.yml ├── .gitignore ├── .golangci.yml ├── AUTHORS ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── PROJECT ├── README.md ├── api └── v1alpha1 │ ├── checkMonitoringCRD.go │ ├── common_types.go │ ├── configuration.go │ ├── connection_types.go │ ├── const.go │ ├── database_types.go │ ├── database_webhook.go │ ├── databasemonitoring_types.go │ ├── databasenodeset_types.go │ ├── groupversion_info.go │ ├── monitoring_types.go │ ├── monitoring_webhook.go │ ├── remotedatabasenodeset_types.go │ ├── remotestoragenodeset_types.go │ ├── service_types.go │ ├── storage_types.go │ ├── storage_webhook.go │ ├── storagemonitoring_types.go │ ├── storagenodeset_types.go │ └── zz_generated.deepcopy.go ├── build └── hack │ └── boilerplate.go.txt ├── cmd └── ydb-kubernetes-operator │ └── main.go ├── deploy └── ydb-operator │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── crds │ ├── database.yaml │ ├── databasemonitoring.yaml │ ├── databasenodeset.yaml │ ├── remotedatabasenodeset.yaml │ ├── remotestoragenodeset.yaml │ ├── storage.yaml │ ├── storagemonitoring.yaml │ └── storagenodeset.yaml │ ├── templates │ ├── _helpers.tpl │ ├── deployment.yaml │ ├── rbac-operator.yaml │ ├── service.yaml │ ├── serviceaccount.yaml │ ├── servicemonitor.yaml │ └── webhooks │ │ ├── certificate.yaml │ │ ├── job-patch │ │ ├── clusterrole.yaml │ │ ├── clusterrolebinding.yaml │ │ ├── job-createSecret.yaml │ │ ├── job-patchWebhook.yaml │ │ ├── role.yaml │ │ ├── rolebinding.yaml │ │ └── serviceaccount.yaml │ │ ├── service.yaml │ │ └── webhooks.yaml │ └── values.yaml ├── docs ├── README.md ├── release-flow.md └── tests.md ├── go.mod ├── go.sum ├── internal ├── annotations │ └── annotations.go ├── cms │ ├── dynconfig.go │ ├── operation.go │ └── tenant.go ├── configuration │ └── schema │ │ ├── configuration.go │ │ ├── domains.go │ │ ├── grpc.go │ │ ├── host.go │ │ ├── key.go │ │ └── schema_test.go ├── connection │ └── connection.go ├── controllers │ ├── constants │ │ └── constants.go │ ├── database │ │ ├── controller.go │ │ ├── controller_test.go │ │ ├── init.go │ │ └── sync.go │ ├── databasenodeset │ │ ├── controller.go │ │ ├── controller_test.go │ │ └── sync.go │ ├── monitoring │ │ ├── common.go │ │ ├── databasemonitoring_controller.go │ │ ├── monitoring_test.go │ │ └── storagemonitoring_controller.go │ ├── remotedatabasenodeset │ │ ├── controller.go │ │ ├── controller_test.go │ │ ├── remote_objects.go │ │ └── sync.go │ ├── remotestoragenodeset │ │ ├── controller.go │ │ ├── controller_test.go │ │ ├── remote_objects.go │ │ └── sync.go │ ├── storage │ │ ├── config.go │ │ ├── controller.go │ │ ├── controller_test.go │ │ ├── init.go │ │ └── sync.go │ └── storagenodeset │ │ ├── controller.go │ │ ├── controller_test.go │ │ └── sync.go ├── encryption │ └── key.go ├── exec │ └── exec.go ├── healthcheck │ └── healthcheck.go ├── labels │ ├── label.go │ └── label_test.go ├── metrics │ ├── const.go │ ├── endpoints.go │ └── relabelings.go ├── ptr │ └── ptr.go ├── resources │ ├── configmap.go │ ├── database.go │ ├── database_statefulset.go │ ├── databasenodeset.go │ ├── encryption.go │ ├── predicate.go │ ├── remotedatabasenodeset.go │ ├── remotestoragenodeset.go │ ├── resource.go │ ├── secret.go │ ├── security_context.go │ ├── security_context_test.go │ ├── service.go │ ├── servicemonitor.go │ ├── storage.go │ ├── storage_init_job.go │ ├── storage_statefulset.go │ └── storagenodeset.go └── test │ ├── extra_crds │ └── monitoring.coreos.com_servicemonitors.yaml │ └── k8s_helpers.go ├── samples ├── database.yaml ├── databasemonitoring.yaml ├── kind │ ├── database-3dc.yaml │ ├── database.yaml │ ├── kind-3dc-config.yaml │ ├── kind-config.yaml │ ├── storage-mirror-3dc.yaml │ └── storage.yaml ├── minikube │ ├── database.yaml │ └── storage.yaml ├── remote-rbac.yml ├── storage-block-4-2.yaml ├── storage-mirror-3dc-nodeset.yaml ├── storage-mirror-3dc.yaml └── storagemonitoring.yaml └── tests ├── cfg ├── kind-cluster-config.yaml ├── operator-local-values.yaml └── operator-values.yaml ├── compatibility └── compatibility_suite_test.go ├── data ├── database.crt ├── database.key ├── generate-crts │ ├── README.md │ ├── ca.crt │ ├── ca.key │ ├── ca.srl │ └── generate-test-certs.sh ├── storage-mirror-3-dc-config-staticCreds.yaml ├── storage-mirror-3-dc-config-tls.yaml ├── storage-mirror-3-dc-config.yaml ├── storage-mirror-3-dc-dynconfig.yaml ├── storage.crt └── storage.key ├── e2e ├── e2e_suite_test.go └── smoke_test.go ├── test-k8s-objects └── objects.go └── test-utils └── test-utils.go /.changes/header.tpl.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | -------------------------------------------------------------------------------- /.changes/unreleased/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/ydb-kubernetes-operator/000f685598f2ec100bdce214b32707f8b046cee0/.changes/unreleased/.gitkeep -------------------------------------------------------------------------------- /.changes/v0.5.31.md: -------------------------------------------------------------------------------- 1 | ## v0.5.31 - 2024-11-04 2 | ### Added 3 | * Initialized a changelog 4 | -------------------------------------------------------------------------------- /.changes/v0.5.32.md: -------------------------------------------------------------------------------- 1 | ## v0.5.32 - 2024-11-05 2 | ### Fixed 3 | * Chart.yaml version is bumped up automatically when a new release PR is created 4 | -------------------------------------------------------------------------------- /.changes/v0.6.0.md: -------------------------------------------------------------------------------- 1 | ## v0.6.0 - 2025-01-29 2 | ### Added 3 | * starting with this release, deploying to dockerhub (ydbplatform/ydb-kubernetes-operator) 4 | * added the ability to create metadata announce for customize dns domain (default: cluster.local) 5 | * new field additionalPodLabels for Storage and Database CRD 6 | * new method buildPodTemplateLabels to append additionalPodLabels for statefulset builders 7 | * compatibility tests running automatically on each new tag 8 | * customize Database and Storage container securityContext 9 | * field externalPort for grpc service to override --grpc-public-port arg 10 | * annotations overrides default secret name and key for arg --auth-token-file 11 | * field ObservedGeneration inside .status.conditions 12 | ### Changed 13 | * up CONTROLLER_GEN_VERSION to 0.16.5 and ENVTEST_VERSION to release-0.17 14 | * refactor package labels to separate methods buildLabels, buildSelectorLabels and buildeNodeSetLabels for each resource 15 | * propagate labels ydb.tech/database-nodeset, ydb.tech/storage-nodeset and ydb.tech/remote-cluster with method makeCommonLabels between resource recasting 16 | ### Fixed 17 | * e2e tests and unit tests flapped because of the race between storage finalizers and uninstalling operator helm chart 18 | * regenerate CRDs in upload-artifacts workflow (as opposed to manually) 19 | * additional kind worker to maintain affinity rules for blobstorage init job 20 | * update the Makefile with the changes in GitHub CI 21 | * bug: missing error handler for arg --auth-token-file 22 | * fix field resourceVersion inside .status.remoteResources.conditions 23 | * panic when create object with .spec.pause is true 24 | * Passing additional secret volumes to blobstorage-init. The init container can now use them without issues. 25 | ### Security 26 | * bump golang-jwt to v4.5.1 (by dependabot) 27 | * bump golang.org/x/net from 0.23.0 to 0.33.0 (by dependabot) 28 | -------------------------------------------------------------------------------- /.changes/v0.6.1.md: -------------------------------------------------------------------------------- 1 | ## v0.6.1 - 2025-02-12 2 | ### Fixed 3 | * fix passing interconnet TLS volume in blobstorage-init job 4 | -------------------------------------------------------------------------------- /.changes/v0.6.2.md: -------------------------------------------------------------------------------- 1 | ## v0.6.2 - 2025-02-24 2 | ### Fixed 3 | * bug: regression with pod name in grpc-public-host arg 4 | -------------------------------------------------------------------------------- /.changes/v0.6.3.md: -------------------------------------------------------------------------------- 1 | ## v0.6.3 - 2025-05-07 2 | -------------------------------------------------------------------------------- /.changes/v0.6.4.md: -------------------------------------------------------------------------------- 1 | ## v0.6.4 - 2025-05-21 2 | ### Added 3 | * `insecurePort` can be specified in GRPC Service spec to create a Service with a second port (for non-tls grpc port in storage) 4 | * Default 2135 port on the GRPC service can now be overridden 5 | ### Fixed 6 | * [development] mutating\validating webhooks now run during medium tests (previously e2e only) 7 | -------------------------------------------------------------------------------- /.changie.yaml: -------------------------------------------------------------------------------- 1 | changesDir: .changes 2 | unreleasedDir: unreleased 3 | headerPath: header.tpl.md 4 | changelogPath: CHANGELOG.md 5 | versionExt: md 6 | versionFormat: '## {{.Version}} - {{.Time.Format "2006-01-02"}}' 7 | kindFormat: '### {{.Kind}}' 8 | changeFormat: '* {{.Body}}' 9 | kinds: 10 | - label: Added 11 | auto: minor 12 | - label: Changed 13 | auto: major 14 | - label: Deprecated 15 | auto: minor 16 | - label: Removed 17 | auto: major 18 | - label: Fixed 19 | auto: patch 20 | - label: Security 21 | auto: patch 22 | newlines: 23 | afterChangelogHeader: 1 24 | beforeChangelogVersion: 1 25 | endOfVersion: 1 26 | envPrefix: CHANGIE_ 27 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | config/ 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/01_BUG_REPORT.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Create a report to help YDB Kubernetes Operator to improve 4 | title: "bug: " 5 | labels: "bug" 6 | assignees: "" 7 | 8 | --- 9 | 10 | # Bug Report 11 | 12 | **YDB Kubernetes Operator version:** 13 | 14 | 15 | 16 | **Environment** 17 | 18 | 19 | 20 | **Current behavior:** 21 | 22 | 23 | 24 | **Expected behavior:** 25 | 26 | 27 | 28 | **Steps to reproduce:** 29 | 30 | 31 | 32 | **Related code:** 33 | 34 | 35 | 36 | ``` 37 | insert short code snippets here 38 | ``` 39 | 40 | **Other information:** 41 | 42 | 43 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/02_FEATURE_REQUEST.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Suggest an idea for this project 4 | title: "feat: " 5 | labels: "enhancement" 6 | assignees: "" 7 | 8 | --- 9 | 10 | # Feature Request 11 | 12 | **Describe the Feature Request** 13 | 14 | 15 | 16 | **Describe Preferred Solution** 17 | 18 | 19 | 20 | **Describe Alternatives** 21 | 22 | 23 | 24 | **Related Code** 25 | 26 | 27 | 28 | **Additional Context** 29 | 30 | 31 | 32 | **If the feature request is approved, would you be willing to submit a PR?** 33 | Yes / No _(Help can be provided if you need assistance submitting a PR)_ 34 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/03_CODEBASE_IMPROVEMENT.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Codebase improvement 3 | about: Provide your feedback for the existing codebase. Suggest a better solution for algorithms, development tools, etc. 4 | title: "dev: " 5 | labels: "enhancement" 6 | assignees: "" 7 | 8 | --- 9 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Pull request type 4 | 5 | 6 | 7 | Please check the type of change your PR introduces: 8 | 9 | - [ ] Bugfix 10 | - [ ] Feature 11 | - [ ] Code style update (formatting, renaming) 12 | - [ ] Refactoring (no functional changes, no api changes) 13 | - [ ] Build related changes 14 | - [ ] Documentation content changes 15 | - [ ] Other (please describe): 16 | 17 | ## What is the current behavior? 18 | 19 | 20 | 21 | Issue Number: N/A 22 | 23 | ## What is the new behavior? 24 | 25 | 26 | 27 | - 28 | - 29 | - 30 | 31 | ## Other information 32 | 33 | 34 | -------------------------------------------------------------------------------- /.github/workflows/create-release-pr.yml: -------------------------------------------------------------------------------- 1 | name: create-release-pr 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | bump_type: 7 | description: 'Which version to bump when creating a release PR: minor or patch?' 8 | required: true 9 | default: 'patch' 10 | type: choice 11 | options: 12 | - patch 13 | - minor 14 | 15 | jobs: 16 | create-release-pr: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: checkout 20 | uses: actions/checkout@v4 21 | 22 | - name: batch-changes 23 | uses: miniscruff/changie-action@v2 24 | with: 25 | version: latest 26 | args: batch ${{ github.event.inputs.bump_type }} 27 | 28 | - name: merge-changes 29 | uses: miniscruff/changie-action@v2 30 | with: 31 | version: latest 32 | args: merge 33 | 34 | - name: print the latest version 35 | id: latest 36 | uses: miniscruff/changie-action@v2 37 | with: 38 | version: latest 39 | args: latest 40 | 41 | - name: print the latest version without "v" 42 | id: latest-no-v 43 | uses: miniscruff/changie-action@v2 44 | with: 45 | version: latest 46 | args: latest --remove-prefix 47 | 48 | - name: bump-chart-version 49 | run: | 50 | VERSION=${{ steps.latest-no-v.outputs.output }} 51 | sed -i "s/^appVersion:.*/appVersion: \"$VERSION\"/" ./deploy/ydb-operator/Chart.yaml 52 | sed -i "s/^version:.*/version: \"$VERSION\"/" ./deploy/ydb-operator/Chart.yaml 53 | 54 | - name: Create Pull Request 55 | uses: peter-evans/create-pull-request@v7 56 | with: 57 | title: Release ${{ steps.latest.outputs.output }} 58 | branch: release/${{ steps.latest.outputs.output }} 59 | commit-message: Release ${{ steps.latest.outputs.output }} 60 | body: | 61 | Here is what a new entry in changelog would look like: 62 | 63 | [`.changes/${{ steps.latest.outputs.output }}.md`](https://github.com/${{ github.repository }}/blob/release/${{ steps.latest.outputs.output }}/.changes/${{ steps.latest.outputs.output }}.md) 64 | token: ${{ github.token }} 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.exe~ 3 | *.dll 4 | *.so 5 | *.dylib 6 | *.test 7 | *.out 8 | *.iws 9 | out/ 10 | .idea/* 11 | .idea_modules/ 12 | atlassian-ide-plugin.xml 13 | com_crashlytics_export_strings.xml 14 | crashlytics.properties 15 | crashlytics-build.properties 16 | fabric.properties 17 | log.json 18 | log.txt 19 | 20 | bin/ 21 | config/ 22 | vendor/ 23 | .envrc 24 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | The following authors have created the source code of "YDB Kubernetes Operator" 2 | published and distributed by YANDEX LLC as the owner: 3 | 4 | Konstantin Bogdanov 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 4 | ## v0.6.4 - 2025-05-21 5 | ### Added 6 | * `insecurePort` can be specified in GRPC Service spec to create a Service with a second port (for non-tls grpc port in storage) 7 | * Default 2135 port on the GRPC service can now be overridden 8 | ### Fixed 9 | * [development] mutating\validating webhooks now run during medium tests (previously e2e only) 10 | 11 | ## v0.6.3 - 2025-05-07 12 | 13 | ## v0.6.2 - 2025-02-24 14 | ### Fixed 15 | * bug: regression with pod name in grpc-public-host arg 16 | 17 | ## v0.6.1 - 2025-02-12 18 | ### Fixed 19 | * fix passing interconnet TLS volume in blobstorage-init job 20 | 21 | ## v0.6.0 - 2025-01-29 22 | ### Added 23 | * starting with this release, deploying to dockerhub (ydbplatform/ydb-kubernetes-operator) 24 | * added the ability to create metadata announce for customize dns domain (default: cluster.local) 25 | * new field additionalPodLabels for Storage and Database CRD 26 | * new method buildPodTemplateLabels to append additionalPodLabels for statefulset builders 27 | * compatibility tests running automatically on each new tag 28 | * customize Database and Storage container securityContext 29 | * field externalPort for grpc service to override --grpc-public-port arg 30 | * annotations overrides default secret name and key for arg --auth-token-file 31 | * field ObservedGeneration inside .status.conditions 32 | ### Changed 33 | * up CONTROLLER_GEN_VERSION to 0.16.5 and ENVTEST_VERSION to release-0.17 34 | * refactor package labels to separate methods buildLabels, buildSelectorLabels and buildeNodeSetLabels for each resource 35 | * propagate labels ydb.tech/database-nodeset, ydb.tech/storage-nodeset and ydb.tech/remote-cluster with method makeCommonLabels between resource recasting 36 | ### Fixed 37 | * e2e tests and unit tests flapped because of the race between storage finalizers and uninstalling operator helm chart 38 | * regenerate CRDs in upload-artifacts workflow (as opposed to manually) 39 | * additional kind worker to maintain affinity rules for blobstorage init job 40 | * update the Makefile with the changes in GitHub CI 41 | * bug: missing error handler for arg --auth-token-file 42 | * fix field resourceVersion inside .status.remoteResources.conditions 43 | * panic when create object with .spec.pause is true 44 | * Passing additional secret volumes to blobstorage-init. The init container can now use them without issues. 45 | ### Security 46 | * bump golang-jwt to v4.5.1 (by dependabot) 47 | * bump golang.org/x/net from 0.23.0 to 0.33.0 (by dependabot) 48 | 49 | ## v0.5.32 - 2024-11-05 50 | ### Fixed 51 | * Chart.yaml version is bumped up automatically when a new release PR is created 52 | 53 | ## v0.5.31 - 2024-11-04 54 | ### Added 55 | * Initialized a changelog 56 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers. All 58 | complaints will be reviewed and investigated and will result in a response that 59 | is deemed necessary and appropriate to the circumstances. The project team is 60 | obligated to maintain confidentiality with regard to the reporter of an incident. 61 | Further details of specific enforcement policies may be posted separately. 62 | 63 | ## Attribution 64 | 65 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 66 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 67 | 68 | [homepage]: https://www.contributor-covenant.org 69 | 70 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Notice to external contributors 2 | 3 | ## Common 4 | 5 | YDB is a free and open project and we appreciate to receive contributions from our community. 6 | 7 | ## Contributing code changes 8 | 9 | If you would like to contribute a new feature or a bug fix, please discuss your idea first on the GitHub issue. 10 | If there is no issue for your idea, please open one. It may be that somebody is already working on it, 11 | or that there are some complex obstacles that you should know about before starting the implementation. 12 | Usually there are several ways to fix a problem and it is important to find the right approach before spending time on a PR 13 | that cannot be merged. 14 | 15 | ## Provide a contribution 16 | 17 | To make a contribution you should submit a pull request. There will probably be discussion about the pull request and, 18 | if any changes are needed, we would love to work with you to get your pull request merged. 19 | 20 | ## Other questions 21 | 22 | If you have any questions, please mail us at info@ydb.tech. 23 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20 as builder 2 | 3 | WORKDIR /workspace 4 | COPY go.mod go.mod 5 | COPY go.sum go.sum 6 | RUN go mod download 7 | 8 | COPY api/ api/ 9 | COPY cmd/ cmd/ 10 | COPY internal/ internal/ 11 | 12 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager cmd/ydb-kubernetes-operator/main.go 13 | 14 | FROM alpine:3.15 15 | WORKDIR / 16 | COPY --from=builder /workspace/manager . 17 | USER 65532:65532 18 | 19 | ENTRYPOINT ["/manager"] 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2021 YANDEX LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | # Code generated by tool. DO NOT EDIT. 2 | # This file is used to track the info used to scaffold your project 3 | # and allow the plugins properly work. 4 | # More info: https://book.kubebuilder.io/reference/project-config.html 5 | domain: ydb.tech 6 | layout: 7 | - go.kubebuilder.io/v3 8 | plugins: 9 | manifests.sdk.operatorframework.io/v2: {} 10 | scorecard.sdk.operatorframework.io/v2: {} 11 | projectName: ydb-operator 12 | repo: github.com/ydb-platform/ydb-operator 13 | resources: 14 | - api: 15 | crdVersion: v1 16 | namespaced: true 17 | controller: true 18 | domain: ydb.tech 19 | group: ydb 20 | kind: Database 21 | path: github.com/ydb-platform/ydb-operator/api/v1alpha1 22 | version: v1alpha1 23 | - api: 24 | crdVersion: v1 25 | namespaced: true 26 | controller: true 27 | domain: ydb.tech 28 | group: ydb 29 | kind: Storage 30 | path: github.com/ydb-platform/ydb-operator/api/v1alpha1 31 | version: v1alpha1 32 | - api: 33 | crdVersion: v1 34 | namespaced: true 35 | controller: true 36 | domain: ydb.tech 37 | group: ydb 38 | kind: DatabaseMonitoring 39 | path: github.com/ydb-platform/ydb-operator/api/v1alpha1 40 | version: v1alpha1 41 | webhooks: 42 | validation: true 43 | webhookVersion: v1 44 | - api: 45 | crdVersion: v1 46 | namespaced: true 47 | controller: true 48 | domain: ydb.tech 49 | group: ydb 50 | kind: StorageMonitoring 51 | path: github.com/ydb-platform/ydb-operator/api/v1alpha1 52 | version: v1alpha1 53 | webhooks: 54 | validation: true 55 | webhookVersion: v1 56 | version: "3" 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![upload-artifacts](https://github.com/ydb-platform/ydb-kubernetes-operator/actions/workflows/upload-artifacts.yml/badge.svg)](https://github.com/ydb-platform/ydb-kubernetes-operator/actions/workflows/upload-artifacts.yml) 2 | [![compatibility-tests](https://github.com/ydb-platform/ydb-kubernetes-operator/actions/workflows/compatibility-tests.yaml/badge.svg)](https://github.com/ydb-platform/ydb-kubernetes-operator/actions/workflows/compatibility-tests.yaml) 3 | 4 | # YDB Kubernetes Operator 5 | 6 | The YDB Kubernetes operator deploys and manages YDB resources in a Kubernetes cluster. 7 | 8 | ## Prerequisites 9 | 10 | 1. Helm 3.1.0+ 11 | 2. Kubernetes 1.20+. 12 | 3. [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 13 | 14 | ## Limitations 15 | 16 | - The Operator currently runs on [Amazon EKS](https://aws.amazon.com/eks/) and [Yandex Managed Service for Kubernetes®](https://cloud.yandex.com/en/services/managed-kubernetes), other cloud providers have not been tested yet. 17 | - The Operator has not been tested with [Istio](https://istio.io/). 18 | 19 | ## Usage 20 | 21 | For steps on how to deploy and use YDB Kubernetes Operator, please refer to [documentation](https://ydb.tech/en/docs/deploy/orchestrated/concepts). 22 | 23 | ## Development 24 | 25 | Refer to the operator [development docs](./docs). 26 | -------------------------------------------------------------------------------- /api/v1alpha1/checkMonitoringCRD.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/go-logr/logr" 7 | kube "k8s.io/client-go/kubernetes" 8 | ctrl "sigs.k8s.io/controller-runtime" 9 | ) 10 | 11 | func checkMonitoringCRD(manager ctrl.Manager, logger logr.Logger, monitoringEnabled bool) error { 12 | if monitoringEnabled { 13 | return nil 14 | } 15 | 16 | config := manager.GetConfig() 17 | clientset, err := kube.NewForConfig(config) 18 | if err != nil { 19 | logger.Error(err, "unable to get clientset while checking monitoring CRD") 20 | return nil 21 | } 22 | _, resources, err := clientset.ServerGroupsAndResources() 23 | if err != nil { 24 | logger.Error(err, "unable to get ServerGroupsAndResources while checking monitoring CRD") 25 | return nil 26 | } 27 | 28 | foundMonitoring := false 29 | for _, resource := range resources { 30 | if resource.GroupVersion == "monitoring.coreos.com/v1" { 31 | for _, res := range resource.APIResources { 32 | if res.Kind == "ServiceMonitor" { 33 | foundMonitoring = true 34 | } 35 | } 36 | } 37 | } 38 | if foundMonitoring { 39 | return nil 40 | } 41 | crdError := fmt.Errorf("required Prometheus CRDs not found in the cluster: `monitoring.coreos.com/v1/servicemonitor`. Please make sure your Prometheus installation is healthy, `kubectl get servicemonitors` must be non-empty") 42 | return crdError 43 | } 44 | -------------------------------------------------------------------------------- /api/v1alpha1/common_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | corev1 "k8s.io/api/core/v1" 5 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 6 | 7 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" 8 | ) 9 | 10 | // NamespacedRef TODO: replace StorageRef 11 | type NamespacedRef struct { 12 | // +kubebuilder:validation:Pattern:=[a-z0-9]([-a-z0-9]*[a-z0-9])? 13 | // +kubebuilder:validation:MaxLength:=63 14 | // +required 15 | Name string `json:"name"` 16 | 17 | // +kubebuilder:validation:Pattern:=[a-z0-9]([-a-z0-9]*[a-z0-9])? 18 | // +kubebuilder:validation:MaxLength:=63 19 | // +optional 20 | Namespace string `json:"namespace"` 21 | } 22 | 23 | // PodImage represents the image information for a container that is used 24 | // to build the StatefulSet. 25 | type PodImage struct { 26 | // Container image with supported YDB version. 27 | // This defaults to the version pinned to the operator and requires a full container and tag/sha name. 28 | // For example: cr.yandex/crptqonuodf51kdj7a7d/ydb:22.2.22 29 | // +optional 30 | Name string `json:"name,omitempty"` 31 | 32 | // (Optional) PullPolicy for the image, which defaults to IfNotPresent. 33 | // Default: IfNotPresent 34 | // +optional 35 | PullPolicyName *corev1.PullPolicy `json:"pullPolicy,omitempty"` 36 | 37 | // (Optional) Secret name containing the dockerconfig to use for a registry that requires authentication. The secret 38 | // must be configured first by the user. 39 | // +optional 40 | PullSecret *string `json:"pullSecret,omitempty"` 41 | } 42 | 43 | type RemoteSpec struct { 44 | // Remote cluster to deploy NodeSet into 45 | // +required 46 | Cluster string `json:"cluster"` 47 | } 48 | 49 | type RemoteResource struct { 50 | Group string `json:"group"` 51 | Version string `json:"version"` 52 | Kind string `json:"kind"` 53 | Name string `json:"name"` 54 | State constants.RemoteResourceState `json:"state"` 55 | Conditions []metav1.Condition `json:"conditions,omitempty"` 56 | } 57 | -------------------------------------------------------------------------------- /api/v1alpha1/connection_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | corev1 "k8s.io/api/core/v1" 5 | ) 6 | 7 | type ConnectionOptions struct { 8 | AccessToken *AccessTokenAuth `json:"accessToken,omitempty"` 9 | StaticCredentials *StaticCredentialsAuth `json:"staticCredentials,omitempty"` 10 | Oauth2TokenExchange *Oauth2TokenExchange `json:"oauth2TokenExchange,omitempty"` 11 | } 12 | 13 | type AccessTokenAuth struct { 14 | *CredentialSource `json:",inline"` 15 | } 16 | 17 | type StaticCredentialsAuth struct { 18 | Username string `json:"username"` 19 | Password *CredentialSource `json:"password,omitempty"` 20 | } 21 | 22 | type Oauth2TokenExchange struct { 23 | Endpoint string `json:"endpoint"` 24 | PrivateKey *CredentialSource `json:"privateKey"` 25 | JWTHeader `json:",inline"` 26 | JWTClaims `json:",inline"` 27 | } 28 | 29 | type JWTHeader struct { 30 | KeyID *string `json:"keyID"` 31 | SignAlg string `json:"signAlg,omitempty"` 32 | } 33 | type JWTClaims struct { 34 | Issuer string `json:"issuer,omitempty"` 35 | Subject string `json:"subject,omitempty"` 36 | Audience string `json:"audience,omitempty"` 37 | ID string `json:"id,omitempty"` 38 | } 39 | 40 | type CredentialSource struct { 41 | SecretKeyRef *corev1.SecretKeySelector `json:"secretKeyRef"` 42 | } 43 | -------------------------------------------------------------------------------- /api/v1alpha1/const.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | const ( 4 | RegistryPath = "cr.yandex/crptqonuodf51kdj7a7d/ydb" 5 | DefaultTag = "22.2.22" 6 | 7 | ImagePathFormat = "%s:%s" 8 | 9 | DefaultDomainName = "cluster.local" 10 | DNSDomainAnnotation = "dns.domain" 11 | 12 | GRPCPort = 2135 13 | GRPCServicePortName = "grpc" 14 | GRPCServiceInsecurePortName = "insecure-grpc" 15 | GRPCProto = "grpc://" 16 | GRPCSProto = "grpcs://" 17 | GRPCServiceFQDNFormat = "%s-grpc.%s.svc.%s" 18 | 19 | InterconnectPort = 19001 20 | InterconnectServicePortName = "interconnect" 21 | InterconnectServiceFQDNFormat = "%s-interconnect.%s.svc.%s" 22 | 23 | StatusPort = 8765 24 | StatusServicePortName = "status" 25 | 26 | DatastreamsPort = 8443 27 | DatastreamsServicePortName = "datastreams" 28 | 29 | DiskPathPrefix = "/dev/kikimr_ssd" 30 | DiskNumberMaxDigits = 2 31 | DiskFilePath = "/data" 32 | 33 | AuthTokenSecretName = "ydb-auth-token-file" 34 | AuthTokenSecretKey = "ydb-auth-token-file" 35 | AuthTokenFileArg = "--auth-token-file" 36 | 37 | DatabaseEncryptionKeySecretDir = "database_encryption" 38 | DatabaseEncryptionKeySecretFile = "key" 39 | DatabaseEncryptionKeyConfigFile = "key.txt" 40 | 41 | ConfigDir = "/opt/ydb/cfg" 42 | ConfigFileName = "config.yaml" 43 | 44 | BinariesDir = "/opt/ydb/bin" 45 | DaemonBinaryName = "ydbd" 46 | 47 | AdditionalSecretsDir = "/opt/ydb/secrets" 48 | AdditionalVolumesDir = "/opt/ydb/volumes" 49 | 50 | DefaultRootUsername = "root" 51 | DefaultRootPassword = "" 52 | DefaultDatabaseDomain = "Root" 53 | DefaultDatabaseEncryptionPin = "EmptyPin" 54 | DefaultSignAlgorithm = "RS256" 55 | 56 | LabelDeploymentKey = "deployment" 57 | LabelDeploymentValueKubernetes = "kubernetes" 58 | LabelSharedDatabaseKey = "shared" 59 | LabelSharedDatabaseValueTrue = "true" 60 | LabelSharedDatabaseValueFalse = "false" 61 | 62 | AnnotationUpdateStrategyOnDelete = "ydb.tech/update-strategy-on-delete" 63 | AnnotationUpdateDNSPolicy = "ydb.tech/update-dns-policy" 64 | AnnotationSkipInitialization = "ydb.tech/skip-initialization" 65 | AnnotationDisableLivenessProbe = "ydb.tech/disable-liveness-probe" 66 | AnnotationDataCenter = "ydb.tech/data-center" 67 | AnnotationGRPCPublicHost = "ydb.tech/grpc-public-host" 68 | AnnotationGRPCPublicPort = "ydb.tech/grpc-public-port" 69 | AnnotationNodeHost = "ydb.tech/node-host" 70 | AnnotationNodeDomain = "ydb.tech/node-domain" 71 | AnnotationAuthTokenSecretName = "ydb.tech/auth-token-secret-name" 72 | AnnotationAuthTokenSecretKey = "ydb.tech/auth-token-secret-key" 73 | 74 | AnnotationValueTrue = "true" 75 | 76 | legacyTenantNameFormat = "/%s/%s" 77 | ) 78 | 79 | type ErasureType string 80 | 81 | const ( 82 | ErasureBlock42 ErasureType = "block-4-2" 83 | ErasureMirror3DC ErasureType = "mirror-3-dc" 84 | None ErasureType = "none" 85 | ) 86 | -------------------------------------------------------------------------------- /api/v1alpha1/databasemonitoring_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // DatabaseMonitoringSpec defines the desired state of DatabaseMonitoring 8 | type DatabaseMonitoringSpec struct { 9 | DatabaseClusterRef NamespacedRef `json:"databaseRef"` 10 | 11 | // (Optional) Additional labels that will be added to the ServiceMonitor 12 | // +optional 13 | AdditionalLabels map[string]string `json:"additionalLabels,omitempty"` 14 | } 15 | 16 | // DatabaseMonitoringStatus defines the observed state of DatabaseMonitoring 17 | type DatabaseMonitoringStatus struct { 18 | State string `json:"state"` 19 | Conditions []metav1.Condition `json:"conditions,omitempty"` 20 | } 21 | 22 | //+kubebuilder:object:root=true 23 | //+kubebuilder:subresource:status 24 | //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.state",description="Monitoring status" 25 | //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" 26 | 27 | // DatabaseMonitoring is the Schema for the databasemonitorings API 28 | type DatabaseMonitoring struct { 29 | metav1.TypeMeta `json:",inline"` 30 | metav1.ObjectMeta `json:"metadata,omitempty"` 31 | 32 | Spec DatabaseMonitoringSpec `json:"spec,omitempty"` 33 | Status DatabaseMonitoringStatus `json:"status,omitempty"` 34 | } 35 | 36 | //+kubebuilder:object:root=true 37 | 38 | // DatabaseMonitoringList contains a list of DatabaseMonitoring 39 | type DatabaseMonitoringList struct { 40 | metav1.TypeMeta `json:",inline"` 41 | metav1.ListMeta `json:"metadata,omitempty"` 42 | Items []DatabaseMonitoring `json:"items"` 43 | } 44 | 45 | func init() { 46 | SchemeBuilder.Register(&DatabaseMonitoring{}, &DatabaseMonitoringList{}) 47 | } 48 | -------------------------------------------------------------------------------- /api/v1alpha1/databasenodeset_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | 6 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" 7 | ) 8 | 9 | // DatabaseNodeSetSpec describes an group nodes of Database object 10 | type DatabaseNodeSetSpec struct { 11 | // YDB Database namespaced reference 12 | // +required 13 | DatabaseRef NamespacedRef `json:"databaseRef"` 14 | 15 | DatabaseClusterSpec `json:",inline"` 16 | 17 | DatabaseNodeSpec `json:",inline"` 18 | } 19 | 20 | // DatabaseNodeSetStatus defines the observed state 21 | type DatabaseNodeSetStatus struct { 22 | State constants.ClusterState `json:"state"` 23 | Conditions []metav1.Condition `json:"conditions,omitempty"` 24 | } 25 | 26 | // DatabaseNodeSetSpecInline describes an group nodes object inside parent object 27 | type DatabaseNodeSetSpecInline struct { 28 | // Name of DatabaseNodeSet object 29 | // +required 30 | Name string `json:"name,omitempty"` 31 | 32 | // Labels for DatabaseNodeSet object 33 | // +optional 34 | Labels map[string]string `json:"labels,omitempty"` 35 | 36 | // Annotations for DatabaseNodeSet object 37 | // +optional 38 | Annotations map[string]string `json:"annotations,omitempty"` 39 | 40 | // (Optional) Object should be reference to RemoteDatabaseNodeSet object 41 | // +optional 42 | Remote *RemoteSpec `json:"remote,omitempty"` 43 | 44 | DatabaseNodeSpec `json:",inline"` 45 | } 46 | 47 | //+kubebuilder:object:root=true 48 | //+kubebuilder:subresource:status 49 | //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.state",description="The status of this DatabaseNodeSet" 50 | //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" 51 | 52 | // DatabaseNodeSet declares StatefulSet parameters for storageRef 53 | type DatabaseNodeSet struct { 54 | metav1.TypeMeta `json:",inline"` 55 | // +optional 56 | metav1.ObjectMeta `json:"metadata,omitempty"` 57 | // +optional 58 | Spec DatabaseNodeSetSpec `json:"spec,omitempty"` 59 | // +optional 60 | // +kubebuilder:default:={state: "Pending"} 61 | Status DatabaseNodeSetStatus `json:"status,omitempty"` 62 | } 63 | 64 | //+kubebuilder:object:root=true 65 | 66 | // DatabaseNodeSetList contains a list of DatabaseNodeSet 67 | type DatabaseNodeSetList struct { 68 | metav1.TypeMeta `json:",inline"` 69 | metav1.ListMeta `json:"metadata,omitempty"` 70 | Items []DatabaseNodeSet `json:"items"` 71 | } 72 | 73 | func init() { 74 | SchemeBuilder.Register(&DatabaseNodeSet{}, &DatabaseNodeSetList{}) 75 | } 76 | 77 | func RecastDatabaseNodeSet(databaseNodeSet *DatabaseNodeSet) *Database { 78 | return &Database{ 79 | ObjectMeta: metav1.ObjectMeta{ 80 | Name: databaseNodeSet.Spec.DatabaseRef.Name, 81 | Namespace: databaseNodeSet.Spec.DatabaseRef.Namespace, 82 | Labels: databaseNodeSet.Labels, 83 | Annotations: databaseNodeSet.Annotations, 84 | }, 85 | Spec: DatabaseSpec{ 86 | DatabaseClusterSpec: databaseNodeSet.Spec.DatabaseClusterSpec, 87 | DatabaseNodeSpec: databaseNodeSet.Spec.DatabaseNodeSpec, 88 | }, 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /api/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | // Package v1alpha1 contains API Schema definitions for the ydb v1alpha1 API group 2 | // +kubebuilder:object:generate=true 3 | // +groupName=ydb.tech 4 | package v1alpha1 5 | 6 | import ( 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | "sigs.k8s.io/controller-runtime/pkg/scheme" 9 | ) 10 | 11 | var ( 12 | // GroupVersion is group version used to register these objects 13 | GroupVersion = schema.GroupVersion{Group: "ydb.tech", Version: "v1alpha1"} 14 | 15 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 16 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 17 | 18 | // AddToScheme adds the types in this group-version to the given scheme. 19 | AddToScheme = SchemeBuilder.AddToScheme 20 | ) 21 | -------------------------------------------------------------------------------- /api/v1alpha1/monitoring_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" 4 | 5 | type MonitoringOptions struct { 6 | Enabled bool `json:"enabled"` 7 | 8 | // Interval at which metrics should be scraped 9 | Interval string `json:"interval,omitempty"` 10 | // RelabelConfig allows dynamic rewriting of the label set, being applied to sample before ingestion. 11 | MetricRelabelings []*v1.RelabelConfig `json:"metricRelabelings,omitempty"` 12 | } 13 | -------------------------------------------------------------------------------- /api/v1alpha1/monitoring_webhook.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/go-logr/logr" 9 | v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" 10 | "k8s.io/apimachinery/pkg/api/errors" 11 | "k8s.io/apimachinery/pkg/runtime" 12 | "k8s.io/apimachinery/pkg/runtime/schema" 13 | "k8s.io/apimachinery/pkg/types" 14 | ctrl "sigs.k8s.io/controller-runtime" 15 | "sigs.k8s.io/controller-runtime/pkg/client" 16 | "sigs.k8s.io/controller-runtime/pkg/client/apiutil" 17 | logf "sigs.k8s.io/controller-runtime/pkg/log" 18 | "sigs.k8s.io/controller-runtime/pkg/runtime/inject" 19 | "sigs.k8s.io/controller-runtime/pkg/webhook" 20 | "sigs.k8s.io/controller-runtime/pkg/webhook/admission" 21 | ) 22 | 23 | // generateValidatePath is a copy from controller-runtime 24 | func generateValidatePath(gvk schema.GroupVersionKind) string { 25 | return "/validate-" + strings.ReplaceAll(gvk.Group, ".", "-") + "-" + 26 | gvk.Version + "-" + strings.ToLower(gvk.Kind) 27 | } 28 | 29 | //+kubebuilder:webhook:path=/validate-ydb-tech-v1alpha1-databasemonitoring,mutating=false,failurePolicy=fail,sideEffects=None,groups=ydb.tech,resources=databasemonitorings,verbs=create,versions=v1alpha1,name=vdatabasemonitoring.kb.io,admissionReviewVersions=v1 30 | //+kubebuilder:webhook:path=/validate-ydb-tech-v1alpha1-storagemonitoring,mutating=false,failurePolicy=fail,sideEffects=None,groups=ydb.tech,resources=storagemonitorings,verbs=create,versions=v1alpha1,name=vstoragemonitoring.kb.io,admissionReviewVersions=v1 31 | 32 | func RegisterMonitoringValidatingWebhook(mgr ctrl.Manager, enableServiceMonitoring bool) error { 33 | // We are using low-level api here because we need pass client to handler 34 | 35 | srv := mgr.GetWebhookServer() 36 | 37 | registerWebHook := func(typ runtime.Object, logName string) error { 38 | gvk, err := apiutil.GVKForObject(typ, mgr.GetScheme()) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | path := generateValidatePath(gvk) 44 | logf.Log.WithName("monitoring-webhooks").Info("Registering a validating webhook", "GVK", gvk, "path", path) 45 | srv.Register(path, &webhook.Admission{ 46 | Handler: &monitoringValidationHandler{ 47 | logger: logf.Log.WithName(logName), 48 | enableServiceMonitoring: enableServiceMonitoring, 49 | mgr: mgr, 50 | }, 51 | }) 52 | return nil 53 | } 54 | 55 | if err := registerWebHook(&DatabaseMonitoring{}, "databasemonitoring-resource"); err != nil { 56 | return err 57 | } 58 | 59 | return registerWebHook(&StorageMonitoring{}, "storagemonitoring-resource") 60 | } 61 | 62 | func ensureNoServiceMonitor(ctx context.Context, client client.Client, namespace string, name string) error { 63 | found := &v1.ServiceMonitor{} 64 | err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, found) 65 | fmt.Printf("err = %v, namespace=%s, name=%s\n", err, namespace, name) 66 | if err == nil { 67 | return fmt.Errorf("service monitor with name %s already exists", name) 68 | } 69 | 70 | if errors.IsNotFound(err) { 71 | return nil 72 | } 73 | 74 | return err 75 | } 76 | 77 | type monitoringValidationHandler struct { 78 | enableServiceMonitoring bool 79 | logger logr.Logger 80 | client client.Client 81 | mgr ctrl.Manager 82 | decoder *admission.Decoder 83 | } 84 | 85 | var ( 86 | _ inject.Client = &monitoringValidationHandler{} 87 | _ admission.Handler = &monitoringValidationHandler{} 88 | ) 89 | 90 | func (v *monitoringValidationHandler) Handle(ctx context.Context, req admission.Request) admission.Response { 91 | if !v.enableServiceMonitoring { 92 | return webhook.Denied("the ydb-operator is running without a service monitoring feature.") 93 | } 94 | 95 | err := checkMonitoringCRD(v.mgr, v.logger, false) 96 | if err != nil { 97 | return webhook.Denied(err.Error()) 98 | } 99 | 100 | // Name can't be an empty string here because it's required in the schema 101 | err = ensureNoServiceMonitor(ctx, v.client, req.Namespace, req.Name) 102 | if err != nil { 103 | return webhook.Denied(err.Error()) 104 | } 105 | 106 | return admission.Allowed("") 107 | } 108 | 109 | func (v *monitoringValidationHandler) InjectClient(c client.Client) error { 110 | v.client = c 111 | return nil 112 | } 113 | 114 | func (v *monitoringValidationHandler) InjectDecoder(d *admission.Decoder) error { 115 | v.decoder = d 116 | return nil 117 | } 118 | -------------------------------------------------------------------------------- /api/v1alpha1/remotedatabasenodeset_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | 6 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" 7 | ) 8 | 9 | //+kubebuilder:object:root=true 10 | //+kubebuilder:subresource:status 11 | //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.state",description="The status of this RemoteDatabaseNodeSet" 12 | //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" 13 | 14 | // RemoteDatabaseNodeSet declares NodeSet spec and status for objects in remote cluster 15 | type RemoteDatabaseNodeSet struct { 16 | metav1.TypeMeta `json:",inline"` 17 | // +optional 18 | metav1.ObjectMeta `json:"metadata,omitempty"` 19 | // +optional 20 | Spec DatabaseNodeSetSpec `json:"spec,omitempty"` 21 | // +optional 22 | // +kubebuilder:default:={state: "Pending"} 23 | Status RemoteDatabaseNodeSetStatus `json:"status,omitempty"` 24 | } 25 | 26 | // DatabaseNodeSetStatus defines the observed state 27 | type RemoteDatabaseNodeSetStatus struct { 28 | State constants.ClusterState `json:"state"` 29 | Conditions []metav1.Condition `json:"conditions,omitempty"` 30 | RemoteResources []RemoteResource `json:"remoteResources,omitempty"` 31 | } 32 | 33 | //+kubebuilder:object:root=true 34 | 35 | // RemoteDatabaseNodeSetList contains a list of RemoteDatabaseNodeSet 36 | type RemoteDatabaseNodeSetList struct { 37 | metav1.TypeMeta `json:",inline"` 38 | metav1.ListMeta `json:"metadata,omitempty"` 39 | Items []RemoteDatabaseNodeSet `json:"items"` 40 | } 41 | 42 | func init() { 43 | SchemeBuilder.Register(&RemoteDatabaseNodeSet{}, &RemoteDatabaseNodeSetList{}) 44 | } 45 | -------------------------------------------------------------------------------- /api/v1alpha1/remotestoragenodeset_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | 6 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" 7 | ) 8 | 9 | //+kubebuilder:object:root=true 10 | //+kubebuilder:subresource:status 11 | //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.state",description="The status of this RemoteStorageNodeSet" 12 | //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" 13 | 14 | // RemoteStorageNodeSet declares NodeSet spec and status for objects in remote cluster 15 | type RemoteStorageNodeSet struct { 16 | metav1.TypeMeta `json:",inline"` 17 | // +optional 18 | metav1.ObjectMeta `json:"metadata,omitempty"` 19 | // +optional 20 | Spec StorageNodeSetSpec `json:"spec,omitempty"` 21 | // +optional 22 | // +kubebuilder:default:={state: "Pending"} 23 | Status RemoteStorageNodeSetStatus `json:"status,omitempty"` 24 | } 25 | 26 | // StorageNodeSetStatus defines the observed state 27 | type RemoteStorageNodeSetStatus struct { 28 | State constants.ClusterState `json:"state"` 29 | Conditions []metav1.Condition `json:"conditions,omitempty"` 30 | RemoteResources []RemoteResource `json:"remoteResources,omitempty"` 31 | } 32 | 33 | //+kubebuilder:object:root=true 34 | 35 | // RemoteStorageNodeSetList contains a list of RemoteStorageNodeSet 36 | type RemoteStorageNodeSetList struct { 37 | metav1.TypeMeta `json:",inline"` 38 | metav1.ListMeta `json:"metadata,omitempty"` 39 | Items []RemoteStorageNodeSet `json:"items"` 40 | } 41 | 42 | func init() { 43 | SchemeBuilder.Register(&RemoteStorageNodeSet{}, &RemoteStorageNodeSetList{}) 44 | } 45 | -------------------------------------------------------------------------------- /api/v1alpha1/service_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import corev1 "k8s.io/api/core/v1" 4 | 5 | type Service struct { 6 | AdditionalLabels map[string]string `json:"additionalLabels,omitempty"` 7 | AdditionalAnnotations map[string]string `json:"additionalAnnotations,omitempty"` 8 | 9 | IPFamilies []corev1.IPFamily `json:"ipFamilies,omitempty"` 10 | IPFamilyPolicy *corev1.IPFamilyPolicyType `json:"ipFamilyPolicy,omitempty"` 11 | } 12 | 13 | type TLSConfiguration struct { 14 | Enabled bool `json:"enabled"` 15 | CertificateAuthority corev1.SecretKeySelector `json:"CA,omitempty"` 16 | Certificate corev1.SecretKeySelector `json:"certificate,omitempty"` 17 | Key corev1.SecretKeySelector `json:"key,omitempty"` // fixme validate: all three or none 18 | } 19 | 20 | type GRPCService struct { 21 | Service `json:""` 22 | 23 | InsecurePort int32 `json:"insecurePort,omitempty"` 24 | Port int32 `json:"port,omitempty"` 25 | 26 | TLSConfiguration *TLSConfiguration `json:"tls,omitempty"` 27 | ExternalHost string `json:"externalHost,omitempty"` 28 | ExternalPort int32 `json:"externalPort,omitempty"` 29 | IPDiscovery *IPDiscovery `json:"ipDiscovery,omitempty"` 30 | } 31 | 32 | type InterconnectService struct { 33 | Service `json:""` 34 | 35 | TLSConfiguration *TLSConfiguration `json:"tls,omitempty"` 36 | } 37 | 38 | type StatusService struct { 39 | Service `json:""` 40 | 41 | TLSConfiguration *TLSConfiguration `json:"tls,omitempty"` 42 | } 43 | 44 | type DatastreamsService struct { 45 | Service `json:""` 46 | 47 | TLSConfiguration *TLSConfiguration `json:"tls,omitempty"` 48 | } 49 | 50 | type IPDiscovery struct { 51 | Enabled bool `json:"enabled"` 52 | TargetNameOverride string `json:"targetNameOverride,omitempty"` 53 | IPFamily corev1.IPFamily `json:"ipFamily,omitempty"` 54 | } 55 | -------------------------------------------------------------------------------- /api/v1alpha1/storagemonitoring_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // StorageMonitoringSpec defines the desired state of StorageMonitoring 8 | type StorageMonitoringSpec struct { 9 | StorageRef NamespacedRef `json:"storageRef"` 10 | 11 | // (Optional) Additional labels that will be added to the ServiceMonitor 12 | // +optional 13 | AdditionalLabels map[string]string `json:"additionalLabels,omitempty"` 14 | } 15 | 16 | // StorageMonitoringStatus defines the observed state of StorageMonitoring 17 | type StorageMonitoringStatus struct { 18 | State string `json:"state"` 19 | Conditions []metav1.Condition `json:"conditions,omitempty"` 20 | } 21 | 22 | //+kubebuilder:object:root=true 23 | //+kubebuilder:subresource:status 24 | //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.state",description="Monitoring status" 25 | //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" 26 | 27 | // StorageMonitoring is the Schema for the storagemonitorings API 28 | type StorageMonitoring struct { 29 | metav1.TypeMeta `json:",inline"` 30 | metav1.ObjectMeta `json:"metadata,omitempty"` 31 | 32 | Spec StorageMonitoringSpec `json:"spec,omitempty"` 33 | Status StorageMonitoringStatus `json:"status,omitempty"` 34 | } 35 | 36 | //+kubebuilder:object:root=true 37 | 38 | // StorageMonitoringList contains a list of StorageMonitoring 39 | type StorageMonitoringList struct { 40 | metav1.TypeMeta `json:",inline"` 41 | metav1.ListMeta `json:"metadata,omitempty"` 42 | Items []StorageMonitoring `json:"items"` 43 | } 44 | 45 | func init() { 46 | SchemeBuilder.Register(&StorageMonitoring{}, &StorageMonitoringList{}) 47 | } 48 | -------------------------------------------------------------------------------- /api/v1alpha1/storagenodeset_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | 6 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" 7 | ) 8 | 9 | // StorageNodeSetSpec describes an group nodes of Storage object 10 | type StorageNodeSetSpec struct { 11 | // YDB Storage reference 12 | // +required 13 | StorageRef NamespacedRef `json:"storageRef"` 14 | 15 | StorageClusterSpec `json:",inline"` 16 | 17 | StorageNodeSpec `json:",inline"` 18 | } 19 | 20 | // StorageNodeSetStatus defines the observed state 21 | type StorageNodeSetStatus struct { 22 | State constants.ClusterState `json:"state"` 23 | Conditions []metav1.Condition `json:"conditions,omitempty"` 24 | } 25 | 26 | // StorageNodeSetSpecInline describes an group nodes object inside parent object 27 | type StorageNodeSetSpecInline struct { 28 | // Name of StorageNodeSet object 29 | // +required 30 | Name string `json:"name"` 31 | 32 | // Labels for StorageNodeSet object 33 | // +optional 34 | Labels map[string]string `json:"labels,omitempty"` 35 | 36 | // Annotations for StorageNodeSet object 37 | // +optional 38 | Annotations map[string]string `json:"annotations,omitempty"` 39 | 40 | // (Optional) Object should be reference to RemoteStorageNodeSet object 41 | // +optional 42 | Remote *RemoteSpec `json:"remote,omitempty"` 43 | 44 | StorageNodeSpec `json:",inline"` 45 | } 46 | 47 | //+kubebuilder:object:root=true 48 | //+kubebuilder:subresource:status 49 | //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.state",description="The status of this StorageNodeSet" 50 | //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" 51 | 52 | // StorageNodeSet declares StatefulSet parameters 53 | type StorageNodeSet struct { 54 | metav1.TypeMeta `json:",inline"` 55 | // +optional 56 | metav1.ObjectMeta `json:"metadata,omitempty"` 57 | // +optional 58 | Spec StorageNodeSetSpec `json:"spec,omitempty"` 59 | // +optional 60 | // +kubebuilder:default:={state: "Pending"} 61 | Status StorageNodeSetStatus `json:"status,omitempty"` 62 | } 63 | 64 | //+kubebuilder:object:root=true 65 | 66 | // StorageNodeSetList contains a list of StorageNodeSet 67 | type StorageNodeSetList struct { 68 | metav1.TypeMeta `json:",inline"` 69 | metav1.ListMeta `json:"metadata,omitempty"` 70 | Items []StorageNodeSet `json:"items"` 71 | } 72 | 73 | func init() { 74 | SchemeBuilder.Register(&StorageNodeSet{}, &StorageNodeSetList{}) 75 | } 76 | 77 | func RecastStorageNodeSet(storageNodeSet *StorageNodeSet) *Storage { 78 | return &Storage{ 79 | ObjectMeta: metav1.ObjectMeta{ 80 | Name: storageNodeSet.Spec.StorageRef.Name, 81 | Namespace: storageNodeSet.Spec.StorageRef.Namespace, 82 | Labels: storageNodeSet.Labels, 83 | Annotations: storageNodeSet.Annotations, 84 | }, 85 | Spec: StorageSpec{ 86 | StorageClusterSpec: storageNodeSet.Spec.StorageClusterSpec, 87 | StorageNodeSpec: storageNodeSet.Spec.StorageNodeSpec, 88 | }, 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /build/hack/boilerplate.go.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/ydb-kubernetes-operator/000f685598f2ec100bdce214b32707f8b046cee0/build/hack/boilerplate.go.txt -------------------------------------------------------------------------------- /deploy/ydb-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ -------------------------------------------------------------------------------- /deploy/ydb-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: ydb-operator 3 | description: A Helm chart for deploying YDB Kubernetes operator. 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: "0.6.4" 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "0.6.4" 25 | -------------------------------------------------------------------------------- /deploy/ydb-operator/README.md: -------------------------------------------------------------------------------- 1 | # YDB Kubernetes Operator Helm chart 2 | 3 | ## Add repo 4 | 5 | ```console 6 | helm repo add ydb https://charts.ydb.tech 7 | helm repo update 8 | ``` 9 | 10 | _See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ 11 | 12 | ## Install Chart 13 | 14 | ```console 15 | # Helm 16 | $ helm install [RELEASE_NAME] ydb/operator 17 | ``` 18 | 19 | ## Configuration 20 | 21 | See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: 22 | 23 | ```console 24 | helm show values ydb/operator 25 | ``` -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "ydb.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "ydb.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "ydb.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "ydb.labels" -}} 37 | helm.sh/chart: {{ include "ydb.chart" . }} 38 | {{ include "ydb.selectorLabels" . }} 39 | {{- if or (.Chart.AppVersion) (.Values.image.tag) }} 40 | app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "ydb.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "ydb.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | 54 | {{/* 55 | Create webhooks pathPrefix used by service fqdn url 56 | */}} 57 | {{- define "ydb.webhookPathPrefix" -}} 58 | {{- if .Values.webhook.service.enableDefaultPathPrefix -}} 59 | {{- printf "/%s/%s" .Release.Namespace ( include "ydb.fullname" . ) -}} 60 | {{- end }} 61 | {{- if .Values.webhook.service.customPathPrefix -}} 62 | {{- .Values.webhook.service.customPathPrefix -}} 63 | {{- end }} 64 | {{- end -}} -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "ydb.fullname" . }} 5 | labels: 6 | {{- include "ydb.labels" . | nindent 4 }} 7 | spec: 8 | type: {{ .Values.service.type }} 9 | ports: 10 | - port: {{ .Values.service.port }} 11 | targetPort: http 12 | protocol: TCP 13 | name: http 14 | selector: 15 | {{- include "ydb.selectorLabels" . | nindent 4 }} -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: {{ include "ydb.fullname" . }} 5 | labels: 6 | {{- include "ydb.labels" . | nindent 4 }} 7 | -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.metrics.enabled }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | name: {{ include "ydb.fullname" . }} 6 | labels: 7 | {{- include "ydb.labels" . | nindent 4 }} 8 | spec: 9 | endpoints: 10 | - path: /metrics 11 | port: http 12 | selector: 13 | matchLabels: 14 | {{- include "ydb.selectorLabels" . | nindent 6 }} 15 | {{- end }} -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/webhooks/certificate.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.webhook.certManager.enabled -}} 2 | {{- if not .Values.webhook.certManager.issuerRef -}} 3 | # Create a selfsigned Issuer, in order to create a root CA certificate for 4 | # signing webhook serving certificates 5 | apiVersion: cert-manager.io/v1 6 | kind: Issuer 7 | metadata: 8 | name: {{ template "ydb.fullname" . }}-self-signed-issuer 9 | namespace: {{ .Release.Namespace }} 10 | spec: 11 | selfSigned: {} 12 | --- 13 | # Generate a CA Certificate used to sign certificates for the webhook 14 | apiVersion: cert-manager.io/v1 15 | kind: Certificate 16 | metadata: 17 | name: {{ template "ydb.fullname" . }}-root-cert 18 | spec: 19 | secretName: {{ template "ydb.fullname" . }}-root-cert 20 | duration: {{ .Values.webhook.certManager.rootCert.duration | default "43800h0m0s" | quote }} 21 | issuerRef: 22 | name: {{ template "ydb.fullname" . }}-self-signed-issuer 23 | commonName: "ca.webhook.ydb" 24 | isCA: true 25 | --- 26 | # Create an Issuer that uses the above generated CA certificate to issue certs 27 | apiVersion: cert-manager.io/v1 28 | kind: Issuer 29 | metadata: 30 | name: {{ template "ydb.fullname" . }}-root-issuer 31 | spec: 32 | ca: 33 | secretName: {{ template "ydb.fullname" . }}-root-cert 34 | {{- end }} 35 | --- 36 | # generate a server certificate for the apiservices to use 37 | apiVersion: cert-manager.io/v1 38 | kind: Certificate 39 | metadata: 40 | name: {{ template "ydb.fullname" . }}-webhook 41 | spec: 42 | secretName: {{ template "ydb.fullname" . }}-webhook 43 | duration: {{ .Values.webhook.certManager.admissionCert.duration | default "8760h0m0s" | quote }} 44 | issuerRef: 45 | {{- if .Values.webhook.certManager.issuerRef }} 46 | {{- toYaml .Values.webhook.certManager.issuerRef | nindent 4 }} 47 | {{- else }} 48 | name: {{ template "ydb.fullname" . }}-root-issuer 49 | {{- end }} 50 | dnsNames: 51 | {{- if .Values.webhook.service.fqdn }} 52 | - {{ .Values.webhook.service.fqdn }} 53 | {{- end}} 54 | - {{ template "ydb.fullname" . }}-webhook 55 | - {{ template "ydb.fullname" . }}-webhook.{{ .Release.Namespace }} 56 | - {{ template "ydb.fullname" . }}-webhook.{{ .Release.Namespace }}.svc 57 | {{- end -}} 58 | -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/webhooks/job-patch/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.webhook.enabled (not .Values.webhook.certManager.enabled) }} 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: {{ template "ydb.fullname" . }}-webhook 6 | annotations: 7 | "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade 8 | "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded 9 | labels: 10 | app: {{ template "ydb.name" . }}-webhook 11 | {{- include "ydb.labels" . | nindent 4 }} 12 | rules: 13 | - apiGroups: 14 | - admissionregistration.k8s.io 15 | resources: 16 | - validatingwebhookconfigurations 17 | - mutatingwebhookconfigurations 18 | verbs: 19 | - get 20 | - update 21 | {{- end }} 22 | -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/webhooks/job-patch/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.webhook.enabled (not .Values.webhook.certManager.enabled) }} 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRoleBinding 4 | metadata: 5 | name: {{ template "ydb.fullname" . }}-webhook 6 | labels: {{ include "ydb.labels" . | nindent 4 }} 7 | annotations: 8 | "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade 9 | "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded 10 | roleRef: 11 | apiGroup: rbac.authorization.k8s.io 12 | kind: ClusterRole 13 | name: {{ template "ydb.fullname" . }}-webhook 14 | subjects: 15 | - kind: ServiceAccount 16 | name: {{ template "ydb.fullname" . }}-webhook 17 | namespace: {{ .Release.Namespace }} 18 | {{- end }} 19 | -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/webhooks/job-patch/job-createSecret.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.webhook.enabled .Values.webhook.patch.enabled }} 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | name: {{ template "ydb.fullname" . }}-webhook-create 6 | annotations: 7 | "helm.sh/hook": pre-install,pre-upgrade 8 | "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded 9 | labels: 10 | app: {{ template "ydb.name" $ }}-webhook-create 11 | {{- include "ydb.labels" . | nindent 4 }} 12 | spec: 13 | {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} 14 | # Alpha feature since k8s 1.12 15 | ttlSecondsAfterFinished: 0 16 | {{- end }} 17 | template: 18 | metadata: 19 | name: {{ template "ydb.fullname" . }}-webhook-create 20 | {{- with .Values.webhook.patch.podAnnotations }} 21 | annotations: 22 | {{- toYaml . | nindent 8 }} 23 | {{- end }} 24 | labels: 25 | app: {{ template "ydb.name" . }}-webhook-create 26 | {{- include "ydb.labels" . | nindent 8 }} 27 | spec: 28 | {{- if .Values.webhook.patch.priorityClassName }} 29 | priorityClassName: {{ .Values.webhook.patch.priorityClassName }} 30 | {{- end }} 31 | containers: 32 | - name: create 33 | {{- if .Values.webhook.patch.image.sha }} 34 | image: {{ .Values.webhook.patch.image.repository }}:{{ .Values.webhook.patch.image.tag }}@sha256:{{ .Values.webhook.patch.image.sha }} 35 | {{- else }} 36 | image: {{ .Values.webhook.patch.image.repository }}:{{ .Values.webhook.patch.image.tag }} 37 | {{- end }} 38 | imagePullPolicy: {{ .Values.webhook.patch.image.pullPolicy }} 39 | args: 40 | - create 41 | - --host={{ template "ydb.fullname" . }}-webhook,{{ template "ydb.fullname" . }}-webhook.{{ .Release.Namespace }}.svc{{ if .Values.webhook.service.fqdn }},{{ .Values.webhook.service.fqdn }}{{ end }} 42 | - --namespace={{ .Release.Namespace }} 43 | - --secret-name={{ template "ydb.fullname" . }}-webhook 44 | resources: 45 | {{ toYaml .Values.webhook.patch.resources | indent 12 }} 46 | restartPolicy: OnFailure 47 | serviceAccountName: {{ template "ydb.fullname" . }}-webhook 48 | {{- with .Values.webhook.patch.nodeSelector }} 49 | nodeSelector: 50 | {{ toYaml . | indent 8 }} 51 | {{- end }} 52 | {{- with .Values.webhook.patch.affinity }} 53 | affinity: 54 | {{ toYaml . | indent 8 }} 55 | {{- end }} 56 | {{- with .Values.webhook.patch.tolerations }} 57 | tolerations: 58 | {{ toYaml . | indent 8 }} 59 | {{- end }} 60 | {{- if .Values.webhook.patch.securityContext }} 61 | securityContext: 62 | {{ toYaml .Values.webhook.patch.securityContext | indent 8 }} 63 | {{- end }} 64 | {{- end }} 65 | -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/webhooks/job-patch/job-patchWebhook.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.webhook.enabled .Values.webhook.patch.enabled }} 2 | {{- if .Values.webhook.patch.injectCA }} 3 | apiVersion: batch/v1 4 | kind: Job 5 | metadata: 6 | name: {{ template "ydb.fullname" . }}-webhook-patch 7 | annotations: 8 | "helm.sh/hook": post-install,post-upgrade 9 | "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded 10 | labels: 11 | app: {{ template "ydb.name" . }}-webhook-patch 12 | {{- include "ydb.labels" . | nindent 4 }} 13 | spec: 14 | {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} 15 | # Alpha feature since k8s 1.12 16 | ttlSecondsAfterFinished: 0 17 | {{- end }} 18 | template: 19 | metadata: 20 | name: {{ template "ydb.fullname" . }}-webhook-patch 21 | {{- with .Values.webhook.patch.podAnnotations }} 22 | annotations: 23 | {{ toYaml . | indent 8 }} 24 | {{- end }} 25 | labels: 26 | app: {{ template "ydb.name" . }}-webhook-patch 27 | {{- include "ydb.labels" . | nindent 8 }} 28 | spec: 29 | {{- if .Values.webhook.patch.priorityClassName }} 30 | priorityClassName: {{ .Values.webhook.patch.priorityClassName }} 31 | {{- end }} 32 | containers: 33 | - name: patch 34 | {{- if .Values.webhook.patch.image.sha }} 35 | image: {{ .Values.webhook.patch.image.repository }}:{{ .Values.webhook.patch.image.tag }}@sha256:{{ .Values.webhook.patch.image.sha }} 36 | {{- else }} 37 | image: {{ .Values.webhook.patch.image.repository }}:{{ .Values.webhook.patch.image.tag }} 38 | {{- end }} 39 | imagePullPolicy: {{ .Values.webhook.patch.image.pullPolicy }} 40 | args: 41 | - patch 42 | - --webhook-name={{ template "ydb.fullname" . }}-webhook 43 | - --namespace={{ .Release.Namespace }} 44 | - --secret-name={{ template "ydb.fullname" . }}-webhook 45 | resources: 46 | {{ toYaml .Values.webhook.patch.resources | indent 12 }} 47 | restartPolicy: OnFailure 48 | serviceAccountName: {{ template "ydb.fullname" . }}-webhook 49 | {{- with .Values.webhook.patch.nodeSelector }} 50 | nodeSelector: 51 | {{ toYaml . | indent 8 }} 52 | {{- end }} 53 | {{- with .Values.webhook.patch.affinity }} 54 | affinity: 55 | {{ toYaml . | indent 8 }} 56 | {{- end }} 57 | {{- with .Values.webhook.patch.tolerations }} 58 | tolerations: 59 | {{ toYaml . | indent 8 }} 60 | {{- end }} 61 | {{- if .Values.webhook.patch.securityContext }} 62 | securityContext: 63 | {{ toYaml .Values.webhook.patch.securityContext | indent 8 }} 64 | {{- end }} 65 | {{- end }} 66 | {{- end }} 67 | -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/webhooks/job-patch/role.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.webhook.enabled (not .Values.webhook.certManager.enabled) }} 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: {{ template "ydb.fullname" . }}-webhook 6 | labels: {{ include "ydb.labels" . | nindent 4 }} 7 | annotations: 8 | "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade 9 | "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded 10 | rules: 11 | - apiGroups: 12 | - "" 13 | resources: 14 | - secrets 15 | verbs: 16 | - get 17 | - create 18 | {{- end }} 19 | -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/webhooks/job-patch/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.webhook.enabled (not .Values.webhook.certManager.enabled) }} 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: RoleBinding 4 | metadata: 5 | name: {{ template "ydb.fullname" . }}-webhook 6 | labels: {{ include "ydb.labels" . | nindent 4 }} 7 | annotations: 8 | "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade 9 | "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded 10 | roleRef: 11 | apiGroup: rbac.authorization.k8s.io 12 | kind: Role 13 | name: {{ template "ydb.fullname" . }}-webhook 14 | subjects: 15 | - kind: ServiceAccount 16 | name: {{ template "ydb.fullname" . }}-webhook 17 | namespace: {{ .Release.Namespace }} 18 | {{- end }} 19 | -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/webhooks/job-patch/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.webhook.enabled (not .Values.webhook.certManager.enabled) }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ template "ydb.fullname" . }}-webhook 6 | labels: {{ include "ydb.labels" . | nindent 4 }} 7 | annotations: 8 | "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade 9 | "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded 10 | imagePullSecrets: 11 | {{ toYaml .Values.imagePullSecrets | indent 2 }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /deploy/ydb-operator/templates/webhooks/service.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.webhook.enabled }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ include "ydb.fullname" . }}-webhook 6 | labels: 7 | {{- include "ydb.labels" . | nindent 4 }} 8 | spec: 9 | type: {{ .Values.webhook.service.type }} 10 | ports: 11 | - port: {{ .Values.webhook.service.port }} 12 | targetPort: webhook 13 | protocol: TCP 14 | name: webhook 15 | {{- if eq .Values.webhook.service.type "NodePort" }} 16 | nodePort: {{ .Values.webhook.service.nodePort }} 17 | {{- end }} 18 | selector: 19 | {{- include "ydb.selectorLabels" . | nindent 4 }} 20 | {{- end }} 21 | -------------------------------------------------------------------------------- /deploy/ydb-operator/values.yaml: -------------------------------------------------------------------------------- 1 | ## Docker image configuration 2 | ## 3 | image: 4 | ## Operator container pull policy 5 | ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images 6 | ## 7 | pullPolicy: IfNotPresent 8 | repository: cr.yandex/yc/ydb-kubernetes-operator 9 | tag: "REPLACED_BY_CHART_APP_VERSION_IF_UNSPECIFIED" 10 | 11 | ## Secrets to use for Docker registry access 12 | ## Secrets must be provided manually. 13 | ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ 14 | ## Example: 15 | ## pullSecrets: 16 | ## - myRegistryKeySecretName 17 | ## 18 | imagePullSecrets: [] 19 | 20 | nodeSelector: {} 21 | podAnnotations: {} 22 | affinity: {} 23 | tolerations: [] 24 | 25 | nameOverride: "" 26 | fullnameOverride: "" 27 | 28 | ## Resource quotas 29 | ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ 30 | resources: 31 | ## The resource limits for Operator container 32 | ## Example: 33 | ## limits: 34 | ## cpu: 250m 35 | ## memory: 512Mi 36 | limits: {} 37 | ## The requested resources for Operator container 38 | ## Example: 39 | ## requests: 40 | ## cpu: 250m 41 | ## memory: 256Mi 42 | requests: {} 43 | 44 | service: 45 | port: 8080 46 | type: ClusterIP 47 | 48 | metrics: 49 | ## Create ServiceMonitor resources 50 | ## 51 | enabled: false 52 | 53 | mgmtCluster: 54 | ## Watch resources from mgmtCluster 55 | ## 56 | enabled: false 57 | name: "" 58 | ## Define existing kubeconfig Secret name in current namespace 59 | kubeconfig: "remote-kubeconfig" 60 | 61 | webhook: 62 | enabled: true 63 | 64 | service: 65 | type: ClusterIP 66 | port: 9443 67 | ## If type is NodePort: 68 | # nodePort: 9443 69 | # 70 | ## Arbitrary fqdn for WebhookConfiguration instead of a default Service cluster fqdn: 71 | # fqdn: example.org 72 | # 73 | ## PathPrefix for WebhookConfiguration url when fqdn used 74 | ## Set variable to true and use default template / 75 | # enableDefaultPathPrefix: false 76 | ## Instead of default template allowed using your own custom pathPrefix 77 | # customPathPrefix: "/haha" 78 | 79 | 80 | ## If patch enabled, then generate a self-signed certificate for service. 81 | ## When injectCA is true should inject the webhook configurations with generated caBundle. 82 | ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own 83 | ## certs ahead of time if you wish. 84 | ## 85 | patch: 86 | enabled: true 87 | injectCA: true 88 | image: 89 | repository: k8s.gcr.io/ingress-nginx/kube-webhook-certgen 90 | tag: v1.0 91 | pullPolicy: IfNotPresent 92 | resources: {} 93 | ## Provide a priority class name to the webhook patching job 94 | ## 95 | priorityClassName: "" 96 | podAnnotations: {} 97 | nodeSelector: {} 98 | affinity: {} 99 | tolerations: [] 100 | 101 | ## SecurityContext holds pod-level security attributes and common container settings. 102 | ## This defaults to non-root user with uid 2000 and gid 2000. *v1.PodSecurityContext false 103 | ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ 104 | ## 105 | securityContext: 106 | runAsGroup: 2000 107 | runAsNonRoot: true 108 | runAsUser: 2000 109 | 110 | # Use cert-manager to generate webhook certs 111 | certManager: 112 | enabled: false 113 | injectCA: false 114 | # self-signed root certificate 115 | rootCert: 116 | duration: "" # default is 5y 117 | admissionCert: 118 | duration: "" # default is 1y 119 | # issuerRef: 120 | # name: "issuer" 121 | # kind: "ClusterIssuer" 122 | 123 | extraVolumes: [] 124 | extraVolumeMounts: [] 125 | extraInitContainers: [] 126 | extraEnvs: [] 127 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | This document attempts to be some sort of a table of contents: 2 | 3 | 4 | ## Working on a feature 5 | 6 | - don't forget to use `changie` to generate a changelog entry, whenever you complete a feature 7 | - see [here](./tests.md) to learn how to run tests 8 | - see [here](./tests.md) to learn how to release a new version 9 | -------------------------------------------------------------------------------- /docs/release-flow.md: -------------------------------------------------------------------------------- 1 | ## Release flow 2 | 3 | #### How and when the operator version is changed 4 | 5 | The single source of truth is the changelog. 6 | 7 | Currently, version is updated when a developer decides to release a new version by manually invoking 8 | `create-release-pr` workflow in Github Actions: 9 | 10 | - invoke `create-release-pr` workflow 11 | - `changie` tool automatically bumps the version and generates an updated CHANGELOG.md 12 | - if a generated `CHANGELOG.md` looks okay, just merge it 13 | - the `upload-artifacts` workflow will: 14 | - substitute the latest version in `Chart.yaml` 15 | - build artifacts (docker image and helm chart) and upload them to all configured registries 16 | - create a new Github release 17 | -------------------------------------------------------------------------------- /docs/tests.md: -------------------------------------------------------------------------------- 1 | ## Writing tests 2 | 3 | ### Categories 4 | 5 | Tests for the operator have naturally splitted up into three categories: 6 | 7 | #### Small 8 | 9 | These tests are simple unit tests that do not simulate any Kubernetes infrastructure. 10 | Useful for testing basic logic chunks of the operator. 11 | 12 | These tests are located directly next to the files which they are testing. Grep for 13 | `label.go` and `label_test.go` for a simple self-contained example. 14 | 15 | #### Medium 16 | 17 | These tests execute use `sigs.k8s.io/controller-runtime/pkg/envtest` to spin up the 18 | control plane of the Kubernetes cluster. This allows for testing simple interactions 19 | between operator and Kubernetes control plane, such as expecting `StatefulSets`s and 20 | `ConfigMap`s to be created after operator has consumed the `Storage` yaml manifest. 21 | This would not actually spin up physical Pods though, we would test only that all the 22 | resources have persisted in etcd! No real cluster is being created at this point. 23 | 24 | These tests are also located directly next to the files which they are testing. Grep 25 | for `storage/controller_test.go` for an example. 26 | 27 | ##### !! Warning !! while writing medium tests. 28 | 29 | Since `StatefulSet` controller is NOT running (only apiserver and etcd are running in 30 | this lightweight scenario), no `Pod`s will be created and it is useless to try to 31 | `get` pods within such tests. Only the objects that are directly created by the 32 | operator (`StatefulSet`s, `ConfigMap`s and `Secret`s) will be created. If you want to 33 | test some changes in `Pod` template, the correct way is to get the `StatefulSet` 34 | object and query it's `Spec.Template.WhateverYouNeed` field to see the changes 35 | reflected in the pod template of `StatefulSet` itself. Again, refer to 36 | `storage/controller_test.go` for an example. 37 | 38 | #### End to end 39 | 40 | These tests simulate a Kubernetes cluster using [Kind](https://kind.sigs.k8s.io/). 41 | With this framework, it is possible to spin up many kubernetes worker nodes as docker 42 | containers on a single machine. This allows for full-blown smoke tests - apply the 43 | `Storage` manifests, wait until the `Pod`s become available, run `SELECT 1` inside 44 | one of those pods to check that YDB is actually up and running! 45 | 46 | E2E tests are located in [tests/e2e](../tests/e2e) folder. 47 | 48 | ## Running tests 49 | 50 | Currently we run all the tests together all the time, we'll add a snippet on how to 51 | launch tests of fixed size later! 52 | 53 | #### Prerequisites for medium 54 | 55 | Run the following (from the root of the repository): 56 | 57 | ``` 58 | make envtest 59 | ./bin/setup-envtest use 1.26 60 | echo $KUBEBUILDER_ASSETS 61 | ``` 62 | 63 | If you're on Linux this variable should look like this: 64 | `/path/to/.local/share/kubebuilder-envtest/k8s/1.26.1-linux-amd64`. If you're on Mac, 65 | it's probably something similar. 66 | 67 | This snippet will install kube-apiserver and etcd binaries and put them in a location 68 | which the testing framework knows about and will find the binaries during tests. 69 | 70 | #### Prerequisites for end to end 71 | 72 | In order to run end to end tests, you have to install `Kind`. 73 | [Refer to official docs](https://kind.sigs.k8s.io/docs/user/quick-start/#installation). 74 | 75 | Typical snippet to run e2e tests follows: 76 | 77 | ```bash 78 | # In case you had other cluster previously, delete it 79 | kind delete cluster --name=local-kind 80 | 81 | kind create cluster \ 82 | --image=kindest/node:v1.21.14@sha256:9d9eb5fb26b4fbc0c6d95fa8c790414f9750dd583f5d7cee45d92e8c26670aa1 \ 83 | --name=local-kind \ 84 | --config=./tests/cfg/kind-cluster-config.yaml \ 85 | --wait 5m 86 | 87 | # Switch your local kubeconfig to the newly created cluster: 88 | kubectl config use-context kind-local-kind 89 | 90 | # Within tests, the following two images are used: 91 | # cr.yandex/crptqonuodf51kdj7a7d/ydb: 92 | # kind/ydb-operator:current 93 | 94 | # You have to download the ydb image and build the operator image yourself. Then, explicitly 95 | # upload them into the kind cluster. Refer to `./github/workflows/run-tests.yaml` github workflow which essentially 96 | # does the same thing. 97 | kind --name local-kind load docker-image kind/ydb-operator:current 98 | kind --name local-kind load docker-image ydb: 99 | 100 | # Run all tests with disabled concurrency, because there is only one cluster to run tests against 101 | go test -p 1 -v ./... 102 | ``` 103 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ydb-platform/ydb-kubernetes-operator 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/banzaicloud/k8s-objectmatcher v1.7.0 7 | github.com/go-logr/logr v1.2.4 8 | github.com/golang-jwt/jwt/v4 v4.5.1 9 | github.com/google/go-cmp v0.6.0 10 | github.com/onsi/ginkgo/v2 v2.9.4 11 | github.com/onsi/gomega v1.27.6 12 | github.com/pkg/errors v0.9.1 13 | github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.50.0 14 | github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7 15 | github.com/ydb-platform/ydb-go-sdk/v3 v3.74.2 16 | google.golang.org/grpc v1.57.1 17 | google.golang.org/protobuf v1.33.0 18 | gopkg.in/yaml.v3 v3.0.1 19 | k8s.io/api v0.26.15 20 | k8s.io/apimachinery v0.26.15 21 | k8s.io/client-go v0.26.15 22 | k8s.io/kubectl v0.26.15 23 | k8s.io/utils v0.0.0-20230115233650-391b47cb4029 24 | sigs.k8s.io/controller-runtime v0.14.1 25 | ) 26 | 27 | require ( 28 | emperror.dev/errors v0.8.0 // indirect 29 | github.com/beorn7/perks v1.0.1 // indirect 30 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 31 | github.com/davecgh/go-spew v1.1.1 // indirect 32 | github.com/emicklei/go-restful/v3 v3.10.1 // indirect 33 | github.com/evanphx/json-patch v4.12.0+incompatible // indirect 34 | github.com/evanphx/json-patch/v5 v5.6.0 // indirect 35 | github.com/fsnotify/fsnotify v1.6.0 // indirect 36 | github.com/go-logr/zapr v1.2.3 // indirect 37 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 38 | github.com/go-openapi/jsonreference v0.20.2 // indirect 39 | github.com/go-openapi/swag v0.22.3 // indirect 40 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect 41 | github.com/gogo/protobuf v1.3.2 // indirect 42 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 43 | github.com/golang/protobuf v1.5.4 // indirect 44 | github.com/google/gnostic v0.6.9 // indirect 45 | github.com/google/gofuzz v1.2.0 // indirect 46 | github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 // indirect 47 | github.com/google/uuid v1.3.0 // indirect 48 | github.com/imdario/mergo v0.3.13 // indirect 49 | github.com/jonboulle/clockwork v0.3.0 // indirect 50 | github.com/josharian/intern v1.0.0 // indirect 51 | github.com/json-iterator/go v1.1.12 // indirect 52 | github.com/mailru/easyjson v0.7.7 // indirect 53 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 54 | github.com/moby/spdystream v0.2.0 // indirect 55 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 56 | github.com/modern-go/reflect2 v1.0.2 // indirect 57 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 58 | github.com/prometheus/client_golang v1.14.0 // indirect 59 | github.com/prometheus/client_model v0.3.0 // indirect 60 | github.com/prometheus/common v0.39.0 // indirect 61 | github.com/prometheus/procfs v0.9.0 // indirect 62 | github.com/spf13/pflag v1.0.5 // indirect 63 | go.uber.org/atomic v1.7.0 // indirect 64 | go.uber.org/multierr v1.6.0 // indirect 65 | go.uber.org/zap v1.24.0 // indirect 66 | golang.org/x/net v0.33.0 // indirect 67 | golang.org/x/oauth2 v0.7.0 // indirect 68 | golang.org/x/sync v0.10.0 // indirect 69 | golang.org/x/sys v0.28.0 // indirect 70 | golang.org/x/term v0.27.0 // indirect 71 | golang.org/x/text v0.21.0 // indirect 72 | golang.org/x/time v0.3.0 // indirect 73 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect 74 | gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect 75 | google.golang.org/appengine v1.6.7 // indirect 76 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect 77 | gopkg.in/inf.v0 v0.9.1 // indirect 78 | gopkg.in/yaml.v2 v2.4.0 // indirect 79 | k8s.io/apiextensions-apiserver v0.26.15 // indirect 80 | k8s.io/component-base v0.26.15 // indirect 81 | k8s.io/klog/v2 v2.90.0 // indirect 82 | k8s.io/kube-openapi v0.0.0-20230123231816-1cb3ae25d79a // indirect 83 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 84 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 85 | sigs.k8s.io/yaml v1.3.0 // indirect 86 | ) 87 | -------------------------------------------------------------------------------- /internal/annotations/annotations.go: -------------------------------------------------------------------------------- 1 | package annotations 2 | 3 | const ( 4 | PrimaryResourceStorageAnnotation = "ydb.tech/primary-resource-storage" 5 | PrimaryResourceDatabaseAnnotation = "ydb.tech/primary-resource-database" 6 | RemoteResourceVersionAnnotation = "ydb.tech/remote-resource-version" 7 | ConfigurationChecksum = "ydb.tech/configuration-checksum" 8 | StorageFinalizerKey = "ydb.tech/storage-finalizer" 9 | RemoteFinalizerKey = "ydb.tech/remote-finalizer" 10 | LastAppliedAnnotation = "ydb.tech/last-applied" 11 | ) 12 | 13 | func CompareLastAppliedAnnotation(map1, map2 map[string]string) bool { 14 | value1 := getLastAppliedAnnotation(map1) 15 | value2 := getLastAppliedAnnotation(map2) 16 | return value1 == value2 17 | } 18 | 19 | func getLastAppliedAnnotation(annotations map[string]string) string { 20 | for key, value := range annotations { 21 | if key == LastAppliedAnnotation { 22 | return value 23 | } 24 | } 25 | return "" 26 | } 27 | -------------------------------------------------------------------------------- /internal/cms/dynconfig.go: -------------------------------------------------------------------------------- 1 | package cms 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/ydb-platform/ydb-go-genproto/draft/Ydb_DynamicConfig_V1" 9 | "github.com/ydb-platform/ydb-go-genproto/draft/protos/Ydb_DynamicConfig" 10 | "github.com/ydb-platform/ydb-go-genproto/protos/Ydb_Operations" 11 | "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "google.golang.org/protobuf/types/known/durationpb" 13 | "sigs.k8s.io/controller-runtime/pkg/log" 14 | 15 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/connection" 16 | ) 17 | 18 | const ( 19 | GetConfigTimeoutSeconds = 10 20 | ReplaceConfigTimeoutSeconds = 30 21 | ) 22 | 23 | type Config struct { 24 | StorageEndpoint string 25 | Domain string 26 | Config string 27 | Version uint64 28 | DryRun bool 29 | AllowUnknownFields bool 30 | } 31 | 32 | func (c *Config) GetConfig( 33 | ctx context.Context, 34 | opts ...ydb.Option, 35 | ) (*Ydb_DynamicConfig.GetConfigResponse, error) { 36 | logger := log.FromContext(ctx) 37 | 38 | endpoint := fmt.Sprintf("%s/%s", c.StorageEndpoint, c.Domain) 39 | conn, err := connection.Open(ctx, endpoint, ydb.MergeOptions(opts...)) 40 | if err != nil { 41 | return nil, fmt.Errorf("error connecting to YDB: %w", err) 42 | } 43 | defer func() { 44 | connection.Close(ctx, conn) 45 | }() 46 | 47 | cmsCtx, cmsCtxCancel := context.WithTimeout(ctx, GetConfigTimeoutSeconds*time.Second) 48 | defer cmsCtxCancel() 49 | client := Ydb_DynamicConfig_V1.NewDynamicConfigServiceClient(ydb.GRPCConn(conn)) 50 | request := c.makeGetConfigRequest() 51 | 52 | logger.Info("CMS GetConfig request", "endpoint", endpoint, "request", request) 53 | return client.GetConfig(cmsCtx, request) 54 | } 55 | 56 | func (c *Config) ProcessConfigResponse(ctx context.Context, response *Ydb_DynamicConfig.GetConfigResponse) error { 57 | logger := log.FromContext(ctx) 58 | logger.Info("CMS GetConfig response", "response", response) 59 | 60 | configResult := &Ydb_DynamicConfig.GetConfigResult{} 61 | err := response.GetOperation().GetResult().UnmarshalTo(configResult) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | c.Config = configResult.GetConfig() 67 | c.Version = configResult.GetIdentity().GetVersion() 68 | return nil 69 | } 70 | 71 | func (c *Config) ReplaceConfig( 72 | ctx context.Context, 73 | opts ...ydb.Option, 74 | ) (*Ydb_DynamicConfig.ReplaceConfigResponse, error) { 75 | logger := log.FromContext(ctx) 76 | 77 | endpoint := fmt.Sprintf("%s/%s", c.StorageEndpoint, c.Domain) 78 | conn, err := connection.Open(ctx, endpoint, ydb.MergeOptions(opts...)) 79 | if err != nil { 80 | return nil, fmt.Errorf("error connecting to YDB: %w", err) 81 | } 82 | defer func() { 83 | connection.Close(ctx, conn) 84 | }() 85 | 86 | cmsCtx, cmsCtxCancel := context.WithTimeout(ctx, ReplaceConfigTimeoutSeconds*time.Second) 87 | defer cmsCtxCancel() 88 | client := Ydb_DynamicConfig_V1.NewDynamicConfigServiceClient(ydb.GRPCConn(conn)) 89 | request := &Ydb_DynamicConfig.ReplaceConfigRequest{ 90 | Config: c.Config, 91 | DryRun: c.DryRun, 92 | AllowUnknownFields: c.AllowUnknownFields, 93 | } 94 | 95 | logger.Info("CMS ReplaceConfig request", "endpoint", endpoint, "request", request) 96 | return client.ReplaceConfig(cmsCtx, request) 97 | } 98 | 99 | func (c *Config) CheckReplaceConfigResponse(ctx context.Context, response *Ydb_DynamicConfig.ReplaceConfigResponse) (bool, string, error) { 100 | logger := log.FromContext(ctx) 101 | 102 | logger.Info("CMS ReplaceConfig response", "response", response) 103 | return CheckOperationStatus(response.GetOperation()) 104 | } 105 | 106 | func (c *Config) makeGetConfigRequest() *Ydb_DynamicConfig.GetConfigRequest { 107 | request := &Ydb_DynamicConfig.GetConfigRequest{} 108 | request.OperationParams = &Ydb_Operations.OperationParams{ 109 | OperationTimeout: &durationpb.Duration{Seconds: GetConfigTimeoutSeconds}, 110 | } 111 | 112 | return request 113 | } 114 | -------------------------------------------------------------------------------- /internal/cms/operation.go: -------------------------------------------------------------------------------- 1 | package cms 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/ydb-platform/ydb-go-genproto/Ydb_Operation_V1" 9 | "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" 10 | "github.com/ydb-platform/ydb-go-genproto/protos/Ydb_Operations" 11 | "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "sigs.k8s.io/controller-runtime/pkg/log" 13 | 14 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/connection" 15 | ) 16 | 17 | const ( 18 | GetOperationTimeoutSeconds = 10 19 | ) 20 | 21 | type Operation struct { 22 | StorageEndpoint string 23 | Domain string 24 | ID string 25 | } 26 | 27 | func (op *Operation) GetOperation( 28 | ctx context.Context, 29 | opts ...ydb.Option, 30 | ) (*Ydb_Operations.GetOperationResponse, error) { 31 | logger := log.FromContext(ctx) 32 | 33 | endpoint := fmt.Sprintf("%s/%s", op.StorageEndpoint, op.Domain) 34 | conn, err := connection.Open(ctx, endpoint, ydb.MergeOptions(opts...)) 35 | if err != nil { 36 | return nil, fmt.Errorf("error connecting to YDB: %w", err) 37 | } 38 | defer func() { 39 | connection.Close(ctx, conn) 40 | }() 41 | 42 | cmsCtx, cmsCtxCancel := context.WithTimeout(ctx, GetOperationTimeoutSeconds*time.Second) 43 | defer cmsCtxCancel() 44 | client := Ydb_Operation_V1.NewOperationServiceClient(ydb.GRPCConn(conn)) 45 | request := &Ydb_Operations.GetOperationRequest{Id: op.ID} 46 | 47 | logger.Info("CMS GetOperation request", "endpoint", endpoint, "request", request) 48 | return client.GetOperation(cmsCtx, request) 49 | } 50 | 51 | func (op *Operation) CheckGetOperationResponse(ctx context.Context, response *Ydb_Operations.GetOperationResponse) (bool, string, error) { 52 | logger := log.FromContext(ctx) 53 | 54 | logger.Info("CMS GetOperation response", "response", response) 55 | return CheckOperationStatus(response.GetOperation()) 56 | } 57 | 58 | func CheckOperationStatus(operation *Ydb_Operations.Operation) (bool, string, error) { 59 | if operation == nil { 60 | return false, "", ErrEmptyReplyFromStorage 61 | } 62 | 63 | if !operation.GetReady() { 64 | return false, operation.Id, nil 65 | } 66 | 67 | if operation.Status == Ydb.StatusIds_ALREADY_EXISTS || operation.Status == Ydb.StatusIds_SUCCESS { 68 | return true, operation.Id, nil 69 | } 70 | 71 | return true, operation.Id, fmt.Errorf("YDB response error: %v %v", operation.Status, operation.Issues) 72 | } 73 | -------------------------------------------------------------------------------- /internal/cms/tenant.go: -------------------------------------------------------------------------------- 1 | package cms 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/ydb-platform/ydb-go-genproto/Ydb_Cms_V1" 10 | "github.com/ydb-platform/ydb-go-genproto/protos/Ydb_Cms" 11 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "sigs.k8s.io/controller-runtime/pkg/log" 13 | 14 | ydbv1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 15 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/connection" 16 | ) 17 | 18 | const ( 19 | CreateDatabaseTimeoutSeconds = 10 20 | ) 21 | 22 | var ErrEmptyReplyFromStorage = errors.New("empty reply from storage") 23 | 24 | type Tenant struct { 25 | StorageEndpoint string 26 | Domain string 27 | Path string 28 | StorageUnits []ydbv1alpha1.StorageUnit 29 | Shared bool 30 | SharedDatabasePath string 31 | } 32 | 33 | func (t *Tenant) CreateDatabase( 34 | ctx context.Context, 35 | opts ...ydb.Option, 36 | ) (*Ydb_Cms.CreateDatabaseResponse, error) { 37 | logger := log.FromContext(ctx) 38 | 39 | endpoint := fmt.Sprintf("%s/%s", t.StorageEndpoint, t.Domain) 40 | conn, err := connection.Open(ctx, endpoint, opts...) 41 | if err != nil { 42 | return nil, fmt.Errorf("error connecting to YDB: %w", err) 43 | } 44 | defer func() { 45 | connection.Close(ctx, conn) 46 | }() 47 | 48 | cmsCtx, cmsCtxCancel := context.WithTimeout(ctx, CreateDatabaseTimeoutSeconds*time.Second) 49 | defer cmsCtxCancel() 50 | client := Ydb_Cms_V1.NewCmsServiceClient(ydb.GRPCConn(conn)) 51 | request := t.makeCreateDatabaseRequest() 52 | logger.Info("CMS CreateDatabase request", "endpoint", endpoint, "request", request) 53 | return client.CreateDatabase(cmsCtx, request) 54 | } 55 | 56 | func (t *Tenant) CheckCreateDatabaseResponse(ctx context.Context, response *Ydb_Cms.CreateDatabaseResponse) (bool, string, error) { 57 | logger := log.FromContext(ctx) 58 | 59 | logger.Info("CMS CreateDatabase response", "response", response) 60 | return CheckOperationStatus(response.GetOperation()) 61 | } 62 | 63 | func (t *Tenant) makeCreateDatabaseRequest() *Ydb_Cms.CreateDatabaseRequest { 64 | request := &Ydb_Cms.CreateDatabaseRequest{Path: t.Path} 65 | if t.SharedDatabasePath != "" { 66 | request.ResourcesKind = &Ydb_Cms.CreateDatabaseRequest_ServerlessResources{ 67 | ServerlessResources: &Ydb_Cms.ServerlessResources{ 68 | SharedDatabasePath: t.SharedDatabasePath, 69 | }, 70 | } 71 | } else { 72 | storageUnitsPb := []*Ydb_Cms.StorageUnits{} 73 | for _, i := range t.StorageUnits { 74 | storageUnitsPb = append( 75 | storageUnitsPb, 76 | &Ydb_Cms.StorageUnits{UnitKind: i.UnitKind, Count: i.Count}, 77 | ) 78 | } 79 | if t.Shared { 80 | request.ResourcesKind = &Ydb_Cms.CreateDatabaseRequest_SharedResources{ 81 | SharedResources: &Ydb_Cms.Resources{ 82 | StorageUnits: storageUnitsPb, 83 | }, 84 | } 85 | } else { 86 | request.ResourcesKind = &Ydb_Cms.CreateDatabaseRequest_Resources{ 87 | Resources: &Ydb_Cms.Resources{ 88 | StorageUnits: storageUnitsPb, 89 | }, 90 | } 91 | } 92 | } 93 | return request 94 | } 95 | -------------------------------------------------------------------------------- /internal/configuration/schema/configuration.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | type DynConfig struct { 4 | Metadata *Metadata `yaml:"metadata"` 5 | Config map[string]interface{} `yaml:"config"` 6 | AllowedLabels map[string]interface{} `yaml:"allowed_labels"` 7 | SelectorConfig []SelectorConfig `yaml:"selector_config"` 8 | } 9 | type Configuration struct { 10 | DomainsConfig *DomainsConfig `yaml:"domains_config"` 11 | Hosts []Host `yaml:"hosts,omitempty"` 12 | KeyConfig *KeyConfig `yaml:"key_config,omitempty"` 13 | GrpcConfig *GrpcConfig `yaml:"grpc_config,omitempty"` 14 | } 15 | 16 | type Metadata struct { 17 | Kind string `yaml:"kind"` 18 | Cluster string `yaml:"cluster"` 19 | Version uint64 `yaml:"version"` 20 | ID uint64 `yaml:"id,omitempty"` 21 | } 22 | 23 | type SelectorConfig struct { 24 | Description string `yaml:"description"` 25 | Selector map[string]interface{} `yaml:"selector"` 26 | Config map[string]interface{} `yaml:"config"` 27 | } 28 | -------------------------------------------------------------------------------- /internal/configuration/schema/domains.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | type DomainsConfig struct { 4 | SecurityConfig *SecurityConfig `yaml:"security_config,omitempty"` 5 | } 6 | 7 | type SecurityConfig struct { 8 | EnforceUserTokenRequirement *bool `yaml:"enforce_user_token_requirement,omitempty"` 9 | } 10 | -------------------------------------------------------------------------------- /internal/configuration/schema/grpc.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | type GrpcConfig struct { 4 | Port int32 `yaml:"port,omitempty"` 5 | SslPort int32 `yaml:"ssl_port,omitempty"` 6 | } 7 | -------------------------------------------------------------------------------- /internal/configuration/schema/host.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | type Host struct { 4 | Address string `yaml:"address,omitempty"` 5 | Host string `yaml:"host"` 6 | HostConfigID int `yaml:"host_config_id"` 7 | NodeID int `yaml:"node_id"` 8 | Port int `yaml:"port,omitempty"` 9 | WalleLocation WalleLocation `yaml:"walle_location,omitempty"` 10 | } 11 | 12 | type WalleLocation struct { 13 | Body int `yaml:"body"` 14 | DataCenter string `yaml:"data_center"` 15 | Rack string `yaml:"rack"` 16 | } 17 | -------------------------------------------------------------------------------- /internal/configuration/schema/key.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | type Key struct { 4 | ContainerPath string `yaml:"container_path"` 5 | ID string `yaml:"id"` 6 | Pin *string `yaml:"pin,omitempty"` 7 | Version int `yaml:"version"` 8 | } 9 | 10 | type KeyConfig struct { 11 | Keys []Key `yaml:"keys"` 12 | } 13 | -------------------------------------------------------------------------------- /internal/connection/connection.go: -------------------------------------------------------------------------------- 1 | package connection 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "errors" 8 | "fmt" 9 | "time" 10 | 11 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "google.golang.org/grpc" 13 | "google.golang.org/grpc/credentials" 14 | "google.golang.org/grpc/credentials/insecure" 15 | "sigs.k8s.io/controller-runtime/pkg/log" 16 | ) 17 | 18 | func Open(ctx context.Context, endpoint string, opts ...ydb.Option) (*ydb.Driver, error) { 19 | ctx, cancel := context.WithTimeout(ctx, 10*time.Second) 20 | defer cancel() 21 | 22 | db, err := ydb.Open( 23 | ctx, 24 | endpoint, 25 | opts..., 26 | ) 27 | if err != nil { 28 | return nil, fmt.Errorf( 29 | "failed to open grpc connection to YDB, endpoint %s. %w", 30 | endpoint, 31 | err, 32 | ) 33 | } 34 | 35 | return db, nil 36 | } 37 | 38 | func Close(ctx context.Context, db *ydb.Driver) { 39 | logger := log.FromContext(ctx) 40 | if err := db.Close(ctx); err != nil { 41 | logger.Error(err, "db close failed") 42 | } 43 | } 44 | 45 | func LoadTLSCredentials(secure bool, caBundle []byte) (grpc.DialOption, error) { 46 | if !secure { 47 | return grpc.WithTransportCredentials(insecure.NewCredentials()), nil 48 | } 49 | var certPool *x509.CertPool 50 | if len(caBundle) > 0 { 51 | certPool = x509.NewCertPool() 52 | if ok := certPool.AppendCertsFromPEM(caBundle); !ok { 53 | return nil, errors.New("failed to parse CA bundle") 54 | } 55 | } else { 56 | var err error 57 | certPool, err = x509.SystemCertPool() 58 | if err != nil { 59 | return nil, fmt.Errorf("failed to get system cert pool, error: %w", err) 60 | } 61 | } 62 | tlsConfig := &tls.Config{ 63 | MinVersion: tls.VersionTLS12, 64 | RootCAs: certPool, 65 | } 66 | return grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), nil 67 | } 68 | -------------------------------------------------------------------------------- /internal/controllers/constants/constants.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | import "time" 4 | 5 | type ( 6 | ClusterState string 7 | RemoteResourceState string 8 | ) 9 | 10 | const ( 11 | StorageKind = "Storage" 12 | StorageNodeSetKind = "StorageNodeSet" 13 | RemoteStorageNodeSetKind = "RemoteStorageNodeSet" 14 | DatabaseKind = "Database" 15 | DatabaseNodeSetKind = "DatabaseNodeSet" 16 | RemoteDatabaseNodeSetKind = "RemoteDatabaseNodeSet" 17 | 18 | // For backward compatibility 19 | OldStorageInitializedCondition = "StorageReady" 20 | OldDatabaseInitializedCondition = "TenantInitialized" 21 | 22 | StoragePreparedCondition = "StoragePrepared" 23 | StorageInitializedCondition = "StorageInitialized" 24 | StorageProvisionedCondition = "StorageProvisioned" 25 | StoragePausedCondition = "StoragePaused" 26 | StorageReadyCondition = "StorageReady" 27 | 28 | DatabasePreparedCondition = "DatabasePrepared" 29 | DatabaseInitializedCondition = "DatabaseInitialized" 30 | DatabaseProvisionedCondition = "DatabaseProvisioned" 31 | DatabasePausedCondition = "DatabasePaused" 32 | DatabaseReadyCondition = "DatabaseReady" 33 | 34 | NodeSetPreparedCondition = "NodeSetPrepared" 35 | NodeSetProvisionedCondition = "NodeSetProvisioned" 36 | NodeSetReadyCondition = "NodeSetReady" 37 | NodeSetPausedCondition = "NodeSetPaused" 38 | 39 | CreateDatabaseOperationCondition = "CreateDatabaseOperation" 40 | ReplaceConfigOperationCondition = "ReplaceConfigOperation" 41 | 42 | ConfigurationSyncedCondition = "ConfigurationSynced" 43 | RemoteResourceSyncedCondition = "ResourceSynced" 44 | 45 | Stop = true 46 | Continue = false 47 | 48 | ReasonInProgress = "InProgress" 49 | ReasonNotRequired = "NotRequired" 50 | ReasonCompleted = "Completed" 51 | ReasonFailed = "Failed" 52 | 53 | DefaultRequeueDelay = 10 * time.Second 54 | StatusUpdateRequeueDelay = 1 * time.Second 55 | ReplaceConfigOperationRequeueDelay = 15 * time.Second 56 | SelfCheckRequeueDelay = 30 * time.Second 57 | StorageInitializationRequeueDelay = 30 * time.Second 58 | DatabaseInitializationRequeueDelay = 30 * time.Second 59 | 60 | DatabasePending ClusterState = "Pending" 61 | DatabasePreparing ClusterState = "Preparing" 62 | DatabaseProvisioning ClusterState = "Provisioning" 63 | DatabaseInitializing ClusterState = "Initializing" 64 | DatabaseReady ClusterState = "Ready" 65 | DatabasePaused ClusterState = "Paused" 66 | 67 | DatabaseNodeSetPending ClusterState = "Pending" 68 | DatabaseNodeSetPreparing ClusterState = "Preparing" 69 | DatabaseNodeSetProvisioning ClusterState = "Provisioning" 70 | DatabaseNodeSetReady ClusterState = "Ready" 71 | DatabaseNodeSetPaused ClusterState = "Paused" 72 | 73 | StoragePending ClusterState = "Pending" 74 | StoragePreparing ClusterState = "Preparing" 75 | StorageProvisioning ClusterState = "Provisioning" 76 | StorageInitializing ClusterState = "Initializing" 77 | StorageReady ClusterState = "Ready" 78 | StoragePaused ClusterState = "Paused" 79 | 80 | StorageNodeSetPending ClusterState = "Pending" 81 | StorageNodeSetPreparing ClusterState = "Preparing" 82 | StorageNodeSetProvisioning ClusterState = "Provisioning" 83 | StorageNodeSetReady ClusterState = "Ready" 84 | StorageNodeSetPaused ClusterState = "Paused" 85 | 86 | ResourceSyncPending RemoteResourceState = "Pending" 87 | ResourceSyncSuccess RemoteResourceState = "Synced" 88 | 89 | StorageAwaitRequeueDelay = 30 * time.Second 90 | SharedDatabaseAwaitRequeueDelay = 30 * time.Second 91 | 92 | OwnerControllerField = ".metadata.controller" 93 | DatabaseRefField = ".spec.databaseRef.name" 94 | StorageRefField = ".spec.storageRef.name" 95 | SecretField = ".spec.secrets" 96 | ) 97 | -------------------------------------------------------------------------------- /internal/controllers/databasenodeset/controller.go: -------------------------------------------------------------------------------- 1 | package databasenodeset 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-logr/logr" 7 | appsv1 "k8s.io/api/apps/v1" 8 | apierrors "k8s.io/apimachinery/pkg/api/errors" 9 | "k8s.io/apimachinery/pkg/runtime" 10 | "k8s.io/client-go/rest" 11 | "k8s.io/client-go/tools/record" 12 | ctrl "sigs.k8s.io/controller-runtime" 13 | "sigs.k8s.io/controller-runtime/pkg/builder" 14 | "sigs.k8s.io/controller-runtime/pkg/client" 15 | "sigs.k8s.io/controller-runtime/pkg/log" 16 | "sigs.k8s.io/controller-runtime/pkg/predicate" 17 | 18 | "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 19 | . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck 20 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" 21 | ) 22 | 23 | // Reconciler reconciles a DatabaseNodeSet object 24 | type Reconciler struct { 25 | client.Client 26 | Recorder record.EventRecorder 27 | Config *rest.Config 28 | Scheme *runtime.Scheme 29 | Log logr.Logger 30 | } 31 | 32 | //+kubebuilder:rbac:groups=ydb.tech,resources=databases,verbs=get;list;watch 33 | //+kubebuilder:rbac:groups=ydb.tech,resources=databasenodesets,verbs=get;list;watch;create;update;patch;delete 34 | //+kubebuilder:rbac:groups=ydb.tech,resources=databasenodesets/status,verbs=get;update;patch 35 | //+kubebuilder:rbac:groups=ydb.tech,resources=databasenodesets/finalizers,verbs=update 36 | //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete 37 | //+kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get;update;patch 38 | //+kubebuilder:rbac:groups=apps,resources=statefulsets/finalizers,verbs=get;list;watch 39 | 40 | // Reconcile is part of the main kubernetes reconciliation loop which aims to 41 | // move the current state of the cluster closer to the desired state. 42 | func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 43 | r.Log = log.FromContext(ctx) 44 | 45 | crDatabaseNodeSet := &v1alpha1.DatabaseNodeSet{} 46 | err := r.Get(ctx, req.NamespacedName, crDatabaseNodeSet) 47 | if err != nil { 48 | if apierrors.IsNotFound(err) { 49 | r.Log.Info("DatabaseNodeSet resource not found") 50 | return ctrl.Result{Requeue: false}, nil 51 | } 52 | r.Log.Error(err, "unable to get DatabaseNodeSet") 53 | return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err 54 | } 55 | 56 | result, err := r.Sync(ctx, crDatabaseNodeSet) 57 | if err != nil { 58 | r.Log.Error(err, "unexpected Sync error") 59 | } 60 | 61 | return result, err 62 | } 63 | 64 | // SetupWithManager sets up the controller with the Manager. 65 | func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { 66 | r.Recorder = mgr.GetEventRecorderFor(DatabaseNodeSetKind) 67 | controller := ctrl.NewControllerManagedBy(mgr) 68 | 69 | return controller. 70 | For(&v1alpha1.DatabaseNodeSet{}, 71 | builder.WithPredicates(predicate.GenerationChangedPredicate{}), 72 | ). 73 | Owns(&appsv1.StatefulSet{}, 74 | builder.WithPredicates(predicate.GenerationChangedPredicate{}), 75 | ). 76 | WithEventFilter(resources.IsDatabaseNodeSetCreatePredicate()). 77 | WithEventFilter(resources.IgnoreDeleteStateUnknownPredicate()). 78 | Complete(r) 79 | } 80 | -------------------------------------------------------------------------------- /internal/controllers/monitoring/common.go: -------------------------------------------------------------------------------- 1 | package monitoring 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/apimachinery/pkg/runtime" 8 | "k8s.io/client-go/tools/record" 9 | ctrl "sigs.k8s.io/controller-runtime" 10 | "sigs.k8s.io/controller-runtime/pkg/client" 11 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 12 | "sigs.k8s.io/controller-runtime/pkg/log" 13 | 14 | api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 15 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" 16 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/metrics" 17 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" 18 | ) 19 | 20 | type Syncer struct { 21 | client.Client 22 | 23 | Object client.Object 24 | Recorder record.EventRecorder 25 | Scheme *runtime.Scheme 26 | 27 | metricsServices []metrics.Service 28 | } 29 | 30 | func (r *Syncer) Sync(ctx context.Context, obj client.Object) (ctrl.Result, error) { 31 | logger := log.FromContext(ctx) 32 | 33 | svc, err := r.findService(ctx, obj) 34 | 35 | if svc == nil { 36 | if err != nil { 37 | r.Recorder.Eventf(r.Object, corev1.EventTypeWarning, "Error", "Unable to find service for db %s/%s", 38 | r.Object.GetNamespace(), r.Object.GetName()) 39 | 40 | log.FromContext(ctx).Error(err, "Unable to find svc for db", "cr", r.Object, "db", obj) 41 | } else { 42 | r.Recorder.Eventf(r.Object, corev1.EventTypeWarning, "Error", "No status service found for db %s/%s. Stop.", 43 | r.Object.GetNamespace(), r.Object.GetName()) 44 | } 45 | return ctrl.Result{}, err 46 | } 47 | 48 | statusServiceLabels := labels.Labels{} 49 | statusServiceLabels.Merge(svc.Labels) 50 | 51 | monitorLabels := labels.Common(obj.GetName(), obj.GetLabels()) 52 | monitorLabels.Merge(r.Object.GetLabels()) 53 | 54 | builder := &resources.ServiceMonitorBuilder{ 55 | Object: obj, 56 | 57 | TargetPort: api.StatusPort, 58 | MetricsServices: r.metricsServices, 59 | Options: &api.MonitoringOptions{}, 60 | 61 | Labels: monitorLabels, 62 | SelectorLabels: statusServiceLabels, 63 | } 64 | 65 | monitor := builder.Placeholder(obj) 66 | monitor.SetName(r.Object.GetName()) 67 | 68 | result, err := resources.CreateOrUpdateOrMaybeIgnore(ctx, r.Client, monitor, func() error { 69 | if err := builder.Build(monitor); err != nil { 70 | r.Recorder.Eventf( 71 | r.Object, 72 | corev1.EventTypeWarning, 73 | "ProvisioningFailed", 74 | "Unable to build resource: %s", err, 75 | ) 76 | return err 77 | } 78 | if err = ctrl.SetControllerReference(r.Object, monitor, r.Scheme); err != nil { 79 | r.Recorder.Eventf( 80 | r.Object, 81 | corev1.EventTypeWarning, 82 | "ProvisioningFailed", 83 | "Unable to set controller reference for resource: %s", err, 84 | ) 85 | return err 86 | } 87 | 88 | return nil 89 | }, func(oldObj, newObj runtime.Object) bool { 90 | return false 91 | }) 92 | 93 | if err != nil { 94 | logger.Error(err, "unexpected Sync error") 95 | } else if result != controllerutil.OperationResultNone { 96 | r.Recorder.Eventf( 97 | r.Object, 98 | corev1.EventTypeNormal, 99 | "Provisioning", 100 | "Update ServiceMonitor %s/%s for %s/%s status = %s", 101 | monitor.GetNamespace(), monitor.GetName(), 102 | obj.GetName(), obj.GetName(), 103 | result, 104 | ) 105 | } 106 | 107 | return ctrl.Result{}, err 108 | } 109 | 110 | func (r *Syncer) findService(ctx context.Context, obj client.Object) (*corev1.Service, error) { 111 | var serviceList corev1.ServiceList 112 | 113 | opts := []client.ListOption{ 114 | client.InNamespace(obj.GetNamespace()), 115 | client.MatchingLabels{labels.ServiceComponent: "status"}, 116 | } 117 | 118 | // we are searching for the database/storage status service and will check ownerReference 119 | if err := r.List(ctx, &serviceList, opts...); err != nil { 120 | r.Recorder.Eventf(obj, corev1.EventTypeWarning, "Syncing", "Unable to get service list: %s", err.Error()) 121 | return nil, err 122 | } 123 | 124 | for _, svc := range serviceList.Items { 125 | for _, owner := range svc.GetOwnerReferences() { 126 | if owner.UID == obj.GetUID() { 127 | return &svc, nil 128 | } 129 | } 130 | } 131 | return nil, nil 132 | } 133 | -------------------------------------------------------------------------------- /internal/controllers/monitoring/databasemonitoring_controller.go: -------------------------------------------------------------------------------- 1 | package monitoring 2 | 3 | import ( 4 | "context" 5 | 6 | monitoring "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" 7 | core "k8s.io/api/core/v1" 8 | apierrs "k8s.io/apimachinery/pkg/api/errors" 9 | "k8s.io/apimachinery/pkg/runtime" 10 | "k8s.io/apimachinery/pkg/types" 11 | "k8s.io/client-go/tools/record" 12 | ctrl "sigs.k8s.io/controller-runtime" 13 | "sigs.k8s.io/controller-runtime/pkg/client" 14 | "sigs.k8s.io/controller-runtime/pkg/log" 15 | 16 | api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 17 | . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck 18 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/metrics" 19 | ) 20 | 21 | // DatabaseMonitoringReconciler reconciles a DatabaseMonitoring object 22 | type DatabaseMonitoringReconciler struct { 23 | client.Client 24 | Recorder record.EventRecorder 25 | Scheme *runtime.Scheme 26 | } 27 | 28 | //+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete 29 | //+kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;patch;delete 30 | //+kubebuilder:rbac:groups=ydb.tech,resources=databasemonitorings,verbs=get;list;watch;create;update;patch;delete 31 | //+kubebuilder:rbac:groups=ydb.tech,resources=databasemonitorings/status,verbs=get;update;patch 32 | //+kubebuilder:rbac:groups=ydb.tech,resources=databasemonitorings/finalizers,verbs=update 33 | 34 | func (r *DatabaseMonitoringReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 35 | logger := log.FromContext(ctx) 36 | 37 | cr := &api.DatabaseMonitoring{} 38 | 39 | if err := r.Get(ctx, req.NamespacedName, cr); err != nil { 40 | if apierrs.IsNotFound(err) { 41 | logger.Info("DatabaseMonitoring has been deleted") 42 | return ctrl.Result{}, nil 43 | } 44 | logger.Error(err, "unable to get DatabaseMonitoring") 45 | return ctrl.Result{}, err 46 | } 47 | 48 | db, err := r.waitForDatabase(ctx, cr) 49 | 50 | if db == nil { 51 | return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err 52 | } 53 | 54 | syncer := &Syncer{ 55 | Client: r.Client, 56 | Recorder: r.Recorder, 57 | Scheme: r.Scheme, 58 | metricsServices: metrics.GetDatabaseMetricsServices(), 59 | Object: cr, 60 | } 61 | return syncer.Sync(ctx, db) 62 | } 63 | 64 | func (r *DatabaseMonitoringReconciler) waitForDatabase(ctx context.Context, cr *api.DatabaseMonitoring) (*api.Database, error) { 65 | ns := cr.Spec.DatabaseClusterRef.Namespace 66 | 67 | if ns == "" { 68 | ns = cr.GetNamespace() 69 | } 70 | 71 | nsName := types.NamespacedName{ 72 | Name: cr.Spec.DatabaseClusterRef.Name, 73 | Namespace: ns, 74 | } 75 | 76 | found := &api.Database{} 77 | 78 | if err := r.Get(ctx, nsName, found); err != nil { 79 | if apierrs.IsNotFound(err) { 80 | r.Recorder.Eventf(cr, core.EventTypeNormal, "Pending", 81 | "Unknown YDB Database cluster %s", 82 | nsName.String()) 83 | return nil, nil 84 | } 85 | r.Recorder.Eventf(cr, core.EventTypeWarning, "Error", 86 | "Unable to find YDB Database %s: %s", nsName.String(), err.Error()) 87 | return nil, err 88 | } else if found.Status.State != DatabaseReady { 89 | r.Recorder.Eventf(cr, core.EventTypeNormal, "Pending", 90 | "YDB Database %s state %s is not ready", 91 | nsName.String(), found.Status.State) 92 | return nil, nil 93 | } 94 | 95 | return found, nil 96 | } 97 | 98 | // SetupWithManager sets up the controller with the Manager. 99 | func (r *DatabaseMonitoringReconciler) SetupWithManager(mgr ctrl.Manager) error { 100 | r.Recorder = mgr.GetEventRecorderFor("DatabaseMonitoring") 101 | 102 | return ctrl.NewControllerManagedBy(mgr). 103 | For(&api.DatabaseMonitoring{}). 104 | Owns(&monitoring.ServiceMonitor{}). 105 | Complete(r) 106 | } 107 | -------------------------------------------------------------------------------- /internal/controllers/monitoring/storagemonitoring_controller.go: -------------------------------------------------------------------------------- 1 | package monitoring 2 | 3 | import ( 4 | "context" 5 | 6 | core "k8s.io/api/core/v1" 7 | apierrs "k8s.io/apimachinery/pkg/api/errors" 8 | "k8s.io/apimachinery/pkg/runtime" 9 | "k8s.io/apimachinery/pkg/types" 10 | "k8s.io/client-go/tools/record" 11 | ctrl "sigs.k8s.io/controller-runtime" 12 | "sigs.k8s.io/controller-runtime/pkg/client" 13 | "sigs.k8s.io/controller-runtime/pkg/log" 14 | 15 | api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 16 | . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck 17 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/metrics" 18 | ) 19 | 20 | // StorageMonitoringReconciler reconciles a StorageMonitoring object 21 | type StorageMonitoringReconciler struct { 22 | client.Client 23 | Recorder record.EventRecorder 24 | Scheme *runtime.Scheme 25 | } 26 | 27 | //+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete 28 | //+kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;patch;delete 29 | //+kubebuilder:rbac:groups=ydb.tech,resources=storagemonitorings,verbs=get;list;watch;create;update;patch;delete 30 | //+kubebuilder:rbac:groups=ydb.tech,resources=storagemonitorings/status,verbs=get;update;patch 31 | //+kubebuilder:rbac:groups=ydb.tech,resources=storagemonitorings/finalizers,verbs=update 32 | 33 | func (r *StorageMonitoringReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 34 | logger := log.FromContext(ctx) 35 | cr := &api.StorageMonitoring{} 36 | 37 | if err := r.Get(ctx, req.NamespacedName, cr); err != nil { 38 | if apierrs.IsNotFound(err) { 39 | logger.Info("StorageMonitoring has been deleted") 40 | return ctrl.Result{}, nil 41 | } 42 | logger.Error(err, "unable to get StorageMonitoring") 43 | return ctrl.Result{}, err 44 | } 45 | 46 | storage, err := r.waitForStorage(ctx, cr) 47 | 48 | if storage == nil { 49 | return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err 50 | } 51 | 52 | syncer := &Syncer{ 53 | Client: r.Client, 54 | Recorder: r.Recorder, 55 | Scheme: r.Scheme, 56 | metricsServices: metrics.GetStorageMetricsServices(), 57 | 58 | Object: cr, 59 | } 60 | return syncer.Sync(ctx, storage) 61 | } 62 | 63 | func (r *StorageMonitoringReconciler) waitForStorage(ctx context.Context, cr *api.StorageMonitoring) (*api.Storage, error) { 64 | ns := cr.Spec.StorageRef.Namespace 65 | 66 | if ns == "" { 67 | ns = cr.GetNamespace() 68 | } 69 | 70 | nsName := types.NamespacedName{ 71 | Name: cr.Spec.StorageRef.Name, 72 | Namespace: ns, 73 | } 74 | 75 | found := &api.Storage{} 76 | 77 | if err := r.Get(ctx, nsName, found); err != nil { 78 | if apierrs.IsNotFound(err) { 79 | r.Recorder.Eventf(cr, core.EventTypeNormal, "Pending", 80 | "Unknown YDB Storage %s", 81 | nsName.String()) 82 | return nil, nil 83 | } 84 | r.Recorder.Eventf(cr, core.EventTypeWarning, "Error", 85 | "Unable to find YDB Storage %s: %s", nsName.String(), err.Error()) 86 | return nil, err 87 | } else if found.Status.State != StorageReady { 88 | r.Recorder.Eventf(cr, core.EventTypeNormal, "Pending", 89 | "YDB Storage %s state %s is not ready", 90 | nsName.String(), found.Status.State) 91 | return nil, nil 92 | } 93 | return found, nil 94 | } 95 | 96 | // SetupWithManager sets up the controller with the Manager. 97 | func (r *StorageMonitoringReconciler) SetupWithManager(mgr ctrl.Manager) error { 98 | r.Recorder = mgr.GetEventRecorderFor("DatabaseMonitoring") 99 | 100 | return ctrl.NewControllerManagedBy(mgr). 101 | For(&api.StorageMonitoring{}). 102 | Complete(r) 103 | } 104 | -------------------------------------------------------------------------------- /internal/controllers/storagenodeset/controller.go: -------------------------------------------------------------------------------- 1 | package storagenodeset 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-logr/logr" 7 | appsv1 "k8s.io/api/apps/v1" 8 | apierrors "k8s.io/apimachinery/pkg/api/errors" 9 | "k8s.io/apimachinery/pkg/runtime" 10 | "k8s.io/client-go/rest" 11 | "k8s.io/client-go/tools/record" 12 | ctrl "sigs.k8s.io/controller-runtime" 13 | "sigs.k8s.io/controller-runtime/pkg/builder" 14 | "sigs.k8s.io/controller-runtime/pkg/client" 15 | "sigs.k8s.io/controller-runtime/pkg/log" 16 | "sigs.k8s.io/controller-runtime/pkg/predicate" 17 | 18 | "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 19 | . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck 20 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" 21 | ) 22 | 23 | // Reconciler reconciles a Storage object 24 | type Reconciler struct { 25 | client.Client 26 | Recorder record.EventRecorder 27 | Config *rest.Config 28 | Scheme *runtime.Scheme 29 | Log logr.Logger 30 | } 31 | 32 | //+kubebuilder:rbac:groups=ydb.tech,resources=storagenodesets,verbs=get;list;watch;create;update;patch;delete 33 | //+kubebuilder:rbac:groups=ydb.tech,resources=storagenodesets/status,verbs=get;update;patch 34 | //+kubebuilder:rbac:groups=ydb.tech,resources=storagenodesets/finalizers,verbs=update 35 | //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete 36 | //+kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get;update;patch 37 | //+kubebuilder:rbac:groups=apps,resources=statefulsets/finalizers,verbs=get;list;watch 38 | 39 | // Reconcile is part of the main kubernetes reconciliation loop which aims to 40 | // move the current state of the cluster closer to the desired state. 41 | func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 42 | r.Log = log.FromContext(ctx) 43 | 44 | crStorageNodeSet := &v1alpha1.StorageNodeSet{} 45 | err := r.Get(ctx, req.NamespacedName, crStorageNodeSet) 46 | if err != nil { 47 | if apierrors.IsNotFound(err) { 48 | r.Log.Info("StorageNodeSet resource not found") 49 | return ctrl.Result{Requeue: false}, nil 50 | } 51 | r.Log.Error(err, "unable to get StorageNodeSet") 52 | return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err 53 | } 54 | 55 | result, err := r.Sync(ctx, crStorageNodeSet) 56 | if err != nil { 57 | r.Log.Error(err, "unexpected Sync error") 58 | } 59 | 60 | return result, err 61 | } 62 | 63 | // SetupWithManager sets up the controller with the Manager. 64 | func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { 65 | r.Recorder = mgr.GetEventRecorderFor(StorageNodeSetKind) 66 | controller := ctrl.NewControllerManagedBy(mgr) 67 | 68 | return controller. 69 | For(&v1alpha1.StorageNodeSet{}, 70 | builder.WithPredicates(predicate.GenerationChangedPredicate{}), 71 | ). 72 | Owns(&appsv1.StatefulSet{}, 73 | builder.WithPredicates(predicate.GenerationChangedPredicate{}), 74 | ). 75 | WithEventFilter(resources.IsStorageNodeSetCreatePredicate()). 76 | WithEventFilter(resources.IgnoreDeleteStateUnknownPredicate()). 77 | Complete(r) 78 | } 79 | -------------------------------------------------------------------------------- /internal/controllers/storagenodeset/controller_test.go: -------------------------------------------------------------------------------- 1 | package storagenodeset_test 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | "strconv" 7 | "testing" 8 | 9 | . "github.com/onsi/ginkgo/v2" 10 | . "github.com/onsi/gomega" 11 | appsv1 "k8s.io/api/apps/v1" 12 | corev1 "k8s.io/api/core/v1" 13 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "sigs.k8s.io/controller-runtime/pkg/client" 15 | "sigs.k8s.io/controller-runtime/pkg/manager" 16 | 17 | "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 18 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storage" 19 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storagenodeset" 20 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/test" 21 | testobjects "github.com/ydb-platform/ydb-kubernetes-operator/tests/test-k8s-objects" 22 | ) 23 | 24 | var ( 25 | k8sClient client.Client 26 | ctx context.Context 27 | ) 28 | 29 | func TestAPIs(t *testing.T) { 30 | RegisterFailHandler(Fail) 31 | 32 | test.SetupK8STestManager(&ctx, &k8sClient, func(mgr *manager.Manager) []test.Reconciler { 33 | return []test.Reconciler{ 34 | &storage.Reconciler{ 35 | Client: k8sClient, 36 | Scheme: (*mgr).GetScheme(), 37 | }, 38 | &storagenodeset.Reconciler{ 39 | Client: k8sClient, 40 | Scheme: (*mgr).GetScheme(), 41 | }, 42 | } 43 | }) 44 | 45 | RunSpecs(t, "StorageNodeSet controller medium tests suite") 46 | } 47 | 48 | var _ = Describe("StorageNodeSet controller medium tests", func() { 49 | var namespace corev1.Namespace 50 | 51 | BeforeEach(func() { 52 | namespace = corev1.Namespace{ 53 | ObjectMeta: metav1.ObjectMeta{ 54 | Name: testobjects.YdbNamespace, 55 | }, 56 | } 57 | Expect(k8sClient.Create(ctx, &namespace)).Should(Succeed()) 58 | }) 59 | 60 | AfterEach(func() { 61 | Expect(k8sClient.Delete(ctx, &namespace)).Should(Succeed()) 62 | }) 63 | 64 | It("Check controller operation through nodeSetSpec inline spec in Storage object", func() { 65 | storageSample := testobjects.DefaultStorage(filepath.Join("..", "..", "..", "tests", "data", "storage-mirror-3-dc-config.yaml")) 66 | 67 | // Test create inline nodeSetSpec in Storage object 68 | testNodeSetName := "nodeset" 69 | storageNodeSetAmount := 3 70 | for idx := 1; idx <= storageNodeSetAmount; idx++ { 71 | storageSample.Spec.NodeSets = append(storageSample.Spec.NodeSets, v1alpha1.StorageNodeSetSpecInline{ 72 | Name: testNodeSetName + "-" + strconv.Itoa(idx), 73 | StorageNodeSpec: v1alpha1.StorageNodeSpec{ 74 | Nodes: 1, 75 | }, 76 | }) 77 | } 78 | Expect(k8sClient.Create(ctx, storageSample)).Should(Succeed()) 79 | 80 | // check that StorageNodeSets was created 81 | storageNodeSets := v1alpha1.StorageNodeSetList{} 82 | Eventually(func() bool { 83 | Expect(k8sClient.List(ctx, &storageNodeSets, client.InNamespace( 84 | testobjects.YdbNamespace, 85 | ))).Should(Succeed()) 86 | foundStorageNodeSet := make(map[int]bool) 87 | for _, storageNodeSet := range storageNodeSets.Items { 88 | for idxNodeSet := 1; idxNodeSet <= storageNodeSetAmount; idxNodeSet++ { 89 | if storageNodeSet.Name == testobjects.StorageName+"-"+testNodeSetName+"-"+strconv.Itoa(idxNodeSet) { 90 | foundStorageNodeSet[idxNodeSet] = true 91 | break 92 | } 93 | } 94 | } 95 | for idxNodeSet := 1; idxNodeSet <= storageNodeSetAmount; idxNodeSet++ { 96 | if !foundStorageNodeSet[idxNodeSet] { 97 | return false 98 | } 99 | } 100 | return true 101 | }, test.Timeout, test.Interval).Should(BeTrue()) 102 | 103 | // check that StatefulSets was created 104 | storageStatefulSets := appsv1.StatefulSetList{} 105 | Eventually(func() bool { 106 | Expect(k8sClient.List(ctx, &storageStatefulSets, client.InNamespace( 107 | testobjects.YdbNamespace, 108 | ))).Should(Succeed()) 109 | foundStatefulSet := make(map[int]bool) 110 | for _, statefulSet := range storageStatefulSets.Items { 111 | for idxNodeSet := 1; idxNodeSet <= storageNodeSetAmount; idxNodeSet++ { 112 | if statefulSet.Name == testobjects.StorageName+"-"+testNodeSetName+"-"+strconv.Itoa(idxNodeSet) { 113 | foundStatefulSet[idxNodeSet] = true 114 | break 115 | } 116 | } 117 | } 118 | for idxNodeSet := 1; idxNodeSet <= storageNodeSetAmount; idxNodeSet++ { 119 | if !foundStatefulSet[idxNodeSet] { 120 | return false 121 | } 122 | } 123 | return true 124 | }, test.Timeout, test.Interval).Should(BeTrue()) 125 | }) 126 | }) 127 | -------------------------------------------------------------------------------- /internal/encryption/key.go: -------------------------------------------------------------------------------- 1 | package encryption 2 | 3 | import ( 4 | "crypto/rand" 5 | "crypto/rsa" 6 | "crypto/x509" 7 | "encoding/pem" 8 | ) 9 | 10 | func GenerateRSAKey(pin string) (string, error) { 11 | rnd := rand.Reader 12 | key, err := rsa.GenerateKey(rnd, 2048) 13 | if err != nil { 14 | return "", err 15 | } 16 | keyBytes, err := x509.MarshalPKCS8PrivateKey(key) 17 | if err != nil { 18 | return "", err 19 | } 20 | block, err := x509.EncryptPEMBlock( 21 | rnd, 22 | "RSA PRIVATE KEY", 23 | keyBytes, 24 | []byte(pin), 25 | x509.PEMCipherAES256, 26 | ) 27 | if err != nil { 28 | return "", err 29 | } 30 | return string(pem.EncodeToMemory(block)), nil 31 | } 32 | -------------------------------------------------------------------------------- /internal/exec/exec.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | 8 | "github.com/pkg/errors" 9 | corev1 "k8s.io/api/core/v1" 10 | "k8s.io/apimachinery/pkg/runtime" 11 | "k8s.io/client-go/kubernetes" 12 | "k8s.io/client-go/rest" 13 | "k8s.io/client-go/tools/remotecommand" 14 | ) 15 | 16 | func InPod( 17 | scheme *runtime.Scheme, 18 | config *rest.Config, 19 | namespace, name, container string, 20 | cmd []string, 21 | ) (string, string, error) { 22 | clientset, err := kubernetes.NewForConfig(config) 23 | if err != nil { 24 | return "", "", errors.Wrapf(err, "failed to create kubernetes clientset") 25 | } 26 | 27 | req := clientset.CoreV1().RESTClient().Post(). 28 | Resource("pods"). 29 | Name(name). 30 | Namespace(namespace). 31 | SubResource("exec") 32 | 33 | parameterCodec := runtime.NewParameterCodec(scheme) 34 | req.VersionedParams(&corev1.PodExecOptions{ 35 | Command: cmd, 36 | Container: container, 37 | Stdin: false, 38 | Stdout: true, 39 | Stderr: true, 40 | TTY: false, 41 | }, parameterCodec) 42 | 43 | exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) 44 | if err != nil { 45 | return "", "", errors.Wrapf(err, "failed to initialize SPDY executor") 46 | } 47 | 48 | var stdout, stderr bytes.Buffer 49 | err = exec.StreamWithContext( 50 | context.TODO(), 51 | remotecommand.StreamOptions{ 52 | Stdin: nil, 53 | Stdout: &stdout, 54 | Stderr: &stderr, 55 | Tty: false, 56 | }, 57 | ) 58 | if err != nil { 59 | return stdout.String(), stderr.String(), errors.Wrapf( 60 | err, 61 | //nolint:govet // TODO @jorres figure out why non-const error messages are not recommended 62 | fmt.Sprintf("failed to stream execution results back, stdout:\n\"%s\"stderr:\n\"%s\"", stdout.String(), stderr.String()), 63 | ) 64 | } 65 | 66 | return stdout.String(), stderr.String(), nil 67 | } 68 | -------------------------------------------------------------------------------- /internal/healthcheck/healthcheck.go: -------------------------------------------------------------------------------- 1 | package healthcheck 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ydb-platform/ydb-go-genproto/Ydb_Monitoring_V1" 8 | "github.com/ydb-platform/ydb-go-genproto/protos/Ydb_Monitoring" 9 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 10 | ydbCredentials "github.com/ydb-platform/ydb-go-sdk/v3/credentials" 11 | "google.golang.org/protobuf/proto" 12 | "sigs.k8s.io/controller-runtime/pkg/log" 13 | 14 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/connection" 15 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" 16 | ) 17 | 18 | func GetSelfCheckResult( 19 | ctx context.Context, 20 | cluster *resources.StorageClusterBuilder, 21 | creds ydbCredentials.Credentials, 22 | opts ...ydb.Option, 23 | ) (*Ydb_Monitoring.SelfCheckResult, error) { 24 | logger := log.FromContext(ctx) 25 | getSelfCheckURL := fmt.Sprintf( 26 | "%s/%s", 27 | cluster.GetStorageEndpointWithProto(), 28 | cluster.Storage.Spec.Domain, 29 | ) 30 | 31 | db, err := connection.Open(ctx, 32 | getSelfCheckURL, 33 | ydb.WithCredentials(creds), 34 | ydb.MergeOptions(opts...), 35 | ) 36 | if err != nil { 37 | return nil, err 38 | } 39 | defer func() { 40 | connection.Close(ctx, db) 41 | }() 42 | 43 | client := Ydb_Monitoring_V1.NewMonitoringServiceClient(ydb.GRPCConn(db)) 44 | response, err := client.SelfCheck(ctx, &Ydb_Monitoring.SelfCheckRequest{}) 45 | if err != nil { 46 | logger.Error(err, "Failed to call SelfCheck") 47 | return nil, err 48 | } 49 | 50 | result := &Ydb_Monitoring.SelfCheckResult{} 51 | if err = proto.Unmarshal(response.Operation.Result.GetValue(), result); err != nil { 52 | logger.Error(err, "Failed to unmarshal SelfCheck response") 53 | return result, err 54 | } 55 | 56 | return result, nil 57 | } 58 | -------------------------------------------------------------------------------- /internal/labels/label.go: -------------------------------------------------------------------------------- 1 | package labels 2 | 3 | // https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/ 4 | const ( 5 | // NameKey The name of a higher level application this one is part of 6 | NameKey = "app.kubernetes.io/name" 7 | // InstanceKey A unique name identifying the instance of an application 8 | InstanceKey = "app.kubernetes.io/instance" 9 | // ComponentKey The component within the architecture 10 | ComponentKey = "app.kubernetes.io/component" 11 | // PartOfKey The name of a higher level application this one is part of 12 | PartOfKey = "app.kubernetes.io/part-of" 13 | // ManagedByKey The tool being used to manage the operation of an application 14 | ManagedByKey = "app.kubernetes.io/managed-by" 15 | 16 | // ServiceComponent The specialization of a Service resource 17 | ServiceComponent = "ydb.tech/service-for" 18 | // StatefulsetComponent The specialization of a Statefulset resource 19 | StatefulsetComponent = "ydb.tech/statefulset-name" 20 | // StorageNodeSetComponent The specialization of a StorageNodeSet resource 21 | StorageNodeSetComponent = "ydb.tech/storage-nodeset" 22 | // DatabaseNodeSetComponent The specialization of a DatabaseNodeSet resource 23 | DatabaseNodeSetComponent = "ydb.tech/database-nodeset" 24 | // RemoteClusterKey The specialization of a remote k8s cluster 25 | RemoteClusterKey = "ydb.tech/remote-cluster" 26 | 27 | StorageComponent = "storage-node" 28 | DynamicComponent = "dynamic-node" 29 | BlobstorageInitComponent = "blobstorage-init" 30 | 31 | GRPCComponent = "grpc" 32 | InterconnectComponent = "interconnect" 33 | StatusComponent = "status" 34 | DatastreamsComponent = "datastreams" 35 | ) 36 | 37 | type Labels map[string]string 38 | 39 | func Common(name string, defaultLabels Labels) Labels { 40 | l := Labels{} 41 | 42 | l.Merge(makeCommonLabels(defaultLabels, name)) 43 | 44 | return l 45 | } 46 | 47 | func (l Labels) AsMap() map[string]string { 48 | return l 49 | } 50 | 51 | func (l Labels) Copy() Labels { 52 | res := Labels{} 53 | 54 | for k, v := range l { 55 | res[k] = v 56 | } 57 | 58 | return res 59 | } 60 | 61 | func (l Labels) Merge(other map[string]string) map[string]string { 62 | if other == nil { 63 | return l 64 | } 65 | 66 | for k, v := range other { 67 | l[k] = v 68 | } 69 | 70 | return l 71 | } 72 | 73 | func makeCommonLabels(other map[string]string, instance string) map[string]string { 74 | common := make(map[string]string) 75 | 76 | // keep part-of customized if it was set by high-level app 77 | var found bool 78 | if common[PartOfKey], found = other[PartOfKey]; !found { 79 | common[PartOfKey] = "yandex-database" 80 | } 81 | 82 | common[NameKey] = "ydb" 83 | common[InstanceKey] = instance 84 | 85 | common[ManagedByKey] = "ydb-operator" 86 | 87 | if storageNodeSetName, exist := other[StorageNodeSetComponent]; exist { 88 | common[StorageNodeSetComponent] = storageNodeSetName 89 | } 90 | 91 | if databaseNodeSetName, exist := other[DatabaseNodeSetComponent]; exist { 92 | common[DatabaseNodeSetComponent] = databaseNodeSetName 93 | } 94 | 95 | if remoteCluster, exist := other[RemoteClusterKey]; exist { 96 | common[RemoteClusterKey] = remoteCluster 97 | } 98 | 99 | return common 100 | } 101 | -------------------------------------------------------------------------------- /internal/labels/label_test.go: -------------------------------------------------------------------------------- 1 | package labels_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | 9 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" 10 | ) 11 | 12 | func TestLabels(t *testing.T) { 13 | RegisterFailHandler(Fail) 14 | RunSpecs(t, "Label suite") 15 | } 16 | 17 | var _ = Describe("Testing labels", func() { 18 | It("merges two sets of labels", func() { 19 | fstLabels := labels.Labels{ 20 | "a": "a", 21 | "b": "b", 22 | } 23 | 24 | sndLabels := labels.Labels{ 25 | "c": "c", 26 | "d": "d", 27 | } 28 | 29 | Expect(fstLabels.Merge(sndLabels)).To(BeEquivalentTo(map[string]string{ 30 | "a": "a", 31 | "b": "b", 32 | "c": "c", 33 | "d": "d", 34 | })) 35 | }) 36 | 37 | It("sets correct defaults", func() { 38 | Expect(labels.Common("ydb", map[string]string{})).To(BeEquivalentTo(map[string]string{ 39 | "app.kubernetes.io/managed-by": "ydb-operator", 40 | "app.kubernetes.io/part-of": "yandex-database", 41 | "app.kubernetes.io/name": "ydb", 42 | "app.kubernetes.io/instance": "ydb", 43 | })) 44 | }) 45 | }) 46 | -------------------------------------------------------------------------------- /internal/metrics/const.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | const ( 4 | MetricEndpointFormat = "/counters/counters=%s/prometheus" 5 | ) 6 | 7 | var storageMetricsServices = []string{ // NB dashes are not allowed in the metric service name 8 | "ydb", 9 | "auth", 10 | "coordinator", 11 | "dsproxy_queue", 12 | "dsproxy", 13 | "grpc", 14 | "pdisks", 15 | "processing", 16 | "proxy", 17 | "followers", 18 | "storage_pool_stat", 19 | "tablets", 20 | "utils", 21 | "dsproxynode", 22 | "interconnect", 23 | "vdisks", 24 | } 25 | 26 | var databaseMetricsServices = []string{ // NB dashes are not allowed in the metric service name 27 | "ydb", 28 | "ydb_serverless", 29 | "auth", 30 | "coordinator", 31 | "dsproxy_queue", 32 | "dsproxy", 33 | "grpc", 34 | "kqp", 35 | "processing", 36 | "proxy", 37 | "followers", 38 | "storage_pool_stat", 39 | "streaming", 40 | "tablets", 41 | "utils", 42 | "yql", 43 | "dsproxynode", 44 | "interconnect", 45 | } 46 | -------------------------------------------------------------------------------- /internal/metrics/endpoints.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | 6 | v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" 7 | ) 8 | 9 | type Service struct { 10 | Name string 11 | Path string 12 | Relabelings []*v1.RelabelConfig 13 | } 14 | 15 | func getMetricsServices(services []string) []Service { 16 | metricsServices := make([]Service, 0, len(services)) 17 | for _, serviceName := range services { 18 | var servicePath string 19 | if serviceName == "ydb" || serviceName == "ydb_serverless" { 20 | servicePath = fmt.Sprintf(MetricEndpointFormat, serviceName+"/name_label=name") 21 | } else { 22 | servicePath = fmt.Sprintf(MetricEndpointFormat, serviceName) 23 | } 24 | metricsServices = append(metricsServices, Service{ 25 | Name: serviceName, 26 | Path: servicePath, 27 | Relabelings: GetMetricsRelabelings(serviceName), 28 | }) 29 | } 30 | 31 | return metricsServices 32 | } 33 | 34 | func GetStorageMetricsServices() []Service { 35 | return getMetricsServices(storageMetricsServices) 36 | } 37 | 38 | func GetDatabaseMetricsServices() []Service { 39 | return getMetricsServices(databaseMetricsServices) 40 | } 41 | -------------------------------------------------------------------------------- /internal/metrics/relabelings.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | 6 | v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" 7 | ) 8 | 9 | func GetMetricsRelabelings(metricsService string) []*v1.RelabelConfig { 10 | return []*v1.RelabelConfig{{ 11 | SourceLabels: []string{"__name__"}, 12 | TargetLabel: "__name__", 13 | Regex: "(.*)", 14 | Replacement: fmt.Sprintf("%s_$1", metricsService), 15 | Action: "replace", 16 | }} 17 | } 18 | -------------------------------------------------------------------------------- /internal/ptr/ptr.go: -------------------------------------------------------------------------------- 1 | package ptr 2 | 3 | // Int32 returns pointer to int32 value. 4 | func Int32(v int32) *int32 { 5 | return &v 6 | } 7 | 8 | // Int32 returns pointer to int64 value. 9 | func Int64(v int64) *int64 { 10 | return &v 11 | } 12 | 13 | // Bool returns pointer to bool value. 14 | func Bool(v bool) *bool { 15 | return &v 16 | } 17 | -------------------------------------------------------------------------------- /internal/resources/configmap.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "html/template" 8 | 9 | v1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | 13 | api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 14 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/configuration/schema" 15 | ) 16 | 17 | const keyConfigTmpl = `Keys { 18 | ContainerPath: "{{ .ContainerPath }}" 19 | Pin: "{{ .Pin }}" 20 | Id: "{{ .ID }}" 21 | Version: {{ .Version }} 22 | }` 23 | 24 | type ConfigMapBuilder struct { 25 | client.Object 26 | 27 | Name string 28 | Labels map[string]string 29 | 30 | Data map[string]string 31 | } 32 | 33 | type EncryptionConfigBuilder struct { 34 | client.Object 35 | 36 | Name string 37 | Labels map[string]string 38 | 39 | KeyConfig schema.KeyConfig 40 | } 41 | 42 | func (b *ConfigMapBuilder) Build(obj client.Object) error { 43 | cm, ok := obj.(*v1.ConfigMap) 44 | if !ok { 45 | return errors.New("failed to cast to ConfigMap object") 46 | } 47 | 48 | if cm.ObjectMeta.Name == "" { 49 | cm.ObjectMeta.Name = b.Name 50 | } 51 | cm.ObjectMeta.Namespace = b.GetNamespace() 52 | 53 | cm.Labels = b.Labels 54 | 55 | cm.Data = b.Data 56 | 57 | return nil 58 | } 59 | 60 | func (b *EncryptionConfigBuilder) Build(obj client.Object) error { 61 | cm, ok := obj.(*v1.ConfigMap) 62 | if !ok { 63 | return errors.New("failed to cast to ConfigMap object") 64 | } 65 | 66 | if cm.ObjectMeta.Name == "" { 67 | cm.ObjectMeta.Name = b.Name 68 | } 69 | cm.ObjectMeta.Namespace = b.GetNamespace() 70 | 71 | cm.Labels = b.Labels 72 | 73 | t, err := template.New("keyConfig").Parse(keyConfigTmpl) 74 | if err != nil { 75 | return fmt.Errorf("failed to parse keyConfig template: %w", err) 76 | } 77 | 78 | var buf bytes.Buffer 79 | err = t.Execute(&buf, b.KeyConfig.Keys[0]) 80 | if err != nil { 81 | return fmt.Errorf("failed to execute keyConfig template: %w", err) 82 | } 83 | 84 | cm.Data = map[string]string{api.DatabaseEncryptionKeyConfigFile: buf.String()} 85 | 86 | return nil 87 | } 88 | 89 | func (b *ConfigMapBuilder) Placeholder(cr client.Object) client.Object { 90 | return &v1.ConfigMap{ 91 | ObjectMeta: metav1.ObjectMeta{ 92 | Name: b.Name, 93 | Namespace: cr.GetNamespace(), 94 | }, 95 | } 96 | } 97 | 98 | func (b *EncryptionConfigBuilder) Placeholder(cr client.Object) client.Object { 99 | return &v1.ConfigMap{ 100 | ObjectMeta: metav1.ObjectMeta{ 101 | Name: b.Name, 102 | Namespace: cr.GetNamespace(), 103 | }, 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /internal/resources/databasenodeset.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "errors" 5 | 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | "k8s.io/client-go/rest" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | 10 | api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 11 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" 12 | ) 13 | 14 | type DatabaseNodeSetBuilder struct { 15 | client.Object 16 | 17 | Name string 18 | Labels map[string]string 19 | Annotations map[string]string 20 | 21 | DatabaseNodeSetSpec api.DatabaseNodeSetSpec 22 | } 23 | 24 | type DatabaseNodeSetResource struct { 25 | *api.DatabaseNodeSet 26 | } 27 | 28 | func (b *DatabaseNodeSetBuilder) Build(obj client.Object) error { 29 | dns, ok := obj.(*api.DatabaseNodeSet) 30 | if !ok { 31 | return errors.New("failed to cast to DatabaseNodeSet object") 32 | } 33 | 34 | if dns.ObjectMeta.Name == "" { 35 | dns.ObjectMeta.Name = b.Name 36 | } 37 | dns.ObjectMeta.Namespace = b.GetNamespace() 38 | 39 | dns.ObjectMeta.Labels = b.Labels 40 | dns.ObjectMeta.Annotations = b.Annotations 41 | 42 | dns.Spec = b.DatabaseNodeSetSpec 43 | 44 | return nil 45 | } 46 | 47 | func (b *DatabaseNodeSetBuilder) Placeholder(cr client.Object) client.Object { 48 | return &api.DatabaseNodeSet{ 49 | ObjectMeta: metav1.ObjectMeta{ 50 | Name: b.Name, 51 | Namespace: cr.GetNamespace(), 52 | }, 53 | } 54 | } 55 | 56 | func (b *DatabaseNodeSetResource) GetResourceBuilders(restConfig *rest.Config) []ResourceBuilder { 57 | ydbCr := api.RecastDatabaseNodeSet(b.Unwrap()) 58 | databaseBuilder := NewDatabase(ydbCr) 59 | 60 | statefulSetName := b.Name 61 | statefulSetLabels := databaseBuilder.buildLabels() 62 | statefulSetAnnotations := CopyDict(b.Spec.AdditionalAnnotations) 63 | statefulSetAnnotations[annotations.ConfigurationChecksum] = SHAChecksum(b.Spec.Configuration) 64 | 65 | var resourceBuilders []ResourceBuilder 66 | resourceBuilders = append(resourceBuilders, 67 | &DatabaseStatefulSetBuilder{ 68 | Database: ydbCr, 69 | RestConfig: restConfig, 70 | 71 | Name: statefulSetName, 72 | Labels: statefulSetLabels, 73 | Annotations: statefulSetAnnotations, 74 | }, 75 | ) 76 | return resourceBuilders 77 | } 78 | 79 | func NewDatabaseNodeSet(databaseNodeSet *api.DatabaseNodeSet) DatabaseNodeSetResource { 80 | crDatabaseNodeSet := databaseNodeSet.DeepCopy() 81 | 82 | if crDatabaseNodeSet.Spec.Service.Status.TLSConfiguration == nil { 83 | crDatabaseNodeSet.Spec.Service.Status.TLSConfiguration = &api.TLSConfiguration{Enabled: false} 84 | } 85 | 86 | return DatabaseNodeSetResource{DatabaseNodeSet: crDatabaseNodeSet} 87 | } 88 | 89 | func (b *DatabaseNodeSetResource) Unwrap() *api.DatabaseNodeSet { 90 | return b.DeepCopy() 91 | } 92 | -------------------------------------------------------------------------------- /internal/resources/encryption.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | corev1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "sigs.k8s.io/controller-runtime/pkg/client" 10 | 11 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/encryption" 12 | ) 13 | 14 | type EncryptionSecretBuilder struct { 15 | client.Object 16 | 17 | Labels map[string]string 18 | Pin string 19 | } 20 | 21 | func (b *EncryptionSecretBuilder) Build(obj client.Object) error { 22 | sec, ok := obj.(*corev1.Secret) 23 | if !ok { 24 | return errors.New("failed to cast to Secret object") 25 | } 26 | 27 | if sec.ObjectMeta.Name == "" { 28 | sec.ObjectMeta.Name = b.GetName() 29 | } 30 | sec.ObjectMeta.Namespace = b.GetNamespace() 31 | 32 | sec.Labels = b.Labels 33 | 34 | key, err := encryption.GenerateRSAKey(b.Pin) 35 | if err != nil { 36 | return fmt.Errorf("failed to generate key for encryption: %w", err) 37 | } 38 | 39 | sec.StringData = map[string]string{ 40 | wellKnownNameForEncryptionKeySecret: key, 41 | } 42 | 43 | sec.Type = corev1.SecretTypeOpaque 44 | 45 | return nil 46 | } 47 | 48 | func (b *EncryptionSecretBuilder) Placeholder(cr client.Object) client.Object { 49 | return &corev1.Secret{ 50 | ObjectMeta: metav1.ObjectMeta{ 51 | Name: b.GetName(), 52 | Namespace: cr.GetNamespace(), 53 | }, 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /internal/resources/predicate.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/labels" 5 | "sigs.k8s.io/controller-runtime/pkg/client" 6 | "sigs.k8s.io/controller-runtime/pkg/event" 7 | "sigs.k8s.io/controller-runtime/pkg/predicate" 8 | 9 | api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 10 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" 11 | ) 12 | 13 | func LastAppliedAnnotationPredicate() predicate.Predicate { 14 | return predicate.Funcs{ 15 | UpdateFunc: func(e event.UpdateEvent) bool { 16 | return !annotations.CompareLastAppliedAnnotation( 17 | e.ObjectOld.GetAnnotations(), 18 | e.ObjectNew.GetAnnotations(), 19 | ) 20 | }, 21 | } 22 | } 23 | 24 | func IsStorageCreatePredicate() predicate.Predicate { 25 | return predicate.Funcs{ 26 | CreateFunc: func(e event.CreateEvent) bool { 27 | _, isStorage := e.Object.(*api.Storage) 28 | return isStorage 29 | }, 30 | } 31 | } 32 | 33 | func IsStorageNodeSetCreatePredicate() predicate.Predicate { 34 | return predicate.Funcs{ 35 | CreateFunc: func(e event.CreateEvent) bool { 36 | _, isStorageNodeSet := e.Object.(*api.StorageNodeSet) 37 | return isStorageNodeSet 38 | }, 39 | } 40 | } 41 | 42 | func IsRemoteStorageNodeSetCreatePredicate() predicate.Predicate { 43 | return predicate.Funcs{ 44 | CreateFunc: func(e event.CreateEvent) bool { 45 | _, isRemoteStorageNodeSet := e.Object.(*api.RemoteStorageNodeSet) 46 | return isRemoteStorageNodeSet 47 | }, 48 | } 49 | } 50 | 51 | func IsDatabaseCreatePredicate() predicate.Predicate { 52 | return predicate.Funcs{ 53 | CreateFunc: func(e event.CreateEvent) bool { 54 | _, isDatabase := e.Object.(*api.Database) 55 | return isDatabase 56 | }, 57 | } 58 | } 59 | 60 | func IsDatabaseNodeSetCreatePredicate() predicate.Predicate { 61 | return predicate.Funcs{ 62 | CreateFunc: func(e event.CreateEvent) bool { 63 | _, isDatabaseNodeSet := e.Object.(*api.DatabaseNodeSet) 64 | return isDatabaseNodeSet 65 | }, 66 | } 67 | } 68 | 69 | func IsRemoteDatabaseNodeSetCreatePredicate() predicate.Predicate { 70 | return predicate.Funcs{ 71 | CreateFunc: func(e event.CreateEvent) bool { 72 | _, isRemoteDatabaseNodeSet := e.Object.(*api.RemoteDatabaseNodeSet) 73 | return isRemoteDatabaseNodeSet 74 | }, 75 | } 76 | } 77 | 78 | func IgnoreDeleteStateUnknownPredicate() predicate.Predicate { 79 | return predicate.Funcs{ 80 | DeleteFunc: func(e event.DeleteEvent) bool { 81 | // Evaluates to false if the object has been confirmed deleted. 82 | return !e.DeleteStateUnknown 83 | }, 84 | } 85 | } 86 | 87 | func LabelExistsPredicate(selector labels.Selector) predicate.Predicate { 88 | return predicate.NewPredicateFuncs(func(o client.Object) bool { 89 | return selector.Matches(labels.Set(o.GetLabels())) 90 | }) 91 | } 92 | -------------------------------------------------------------------------------- /internal/resources/secret.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/client-go/kubernetes" 12 | "k8s.io/client-go/rest" 13 | "sigs.k8s.io/controller-runtime/pkg/client" 14 | ) 15 | 16 | func GetSecretKey( 17 | ctx context.Context, 18 | namespace string, 19 | config *rest.Config, 20 | secretKeyRef *corev1.SecretKeySelector, 21 | ) (string, error) { 22 | clientset, err := kubernetes.NewForConfig(config) 23 | if err != nil { 24 | return "", fmt.Errorf("failed to create kubernetes clientset, error: %w", err) 25 | } 26 | 27 | getCtx, cancel := context.WithTimeout(ctx, time.Second) 28 | defer cancel() 29 | secret, err := clientset.CoreV1().Secrets(namespace).Get(getCtx, secretKeyRef.Name, metav1.GetOptions{}) 30 | if err != nil { 31 | return "", fmt.Errorf("failed to get secret %s, error: %w", secretKeyRef.Name, err) 32 | } 33 | 34 | secretVal, exist := secret.Data[secretKeyRef.Key] 35 | if !exist { 36 | errMsg := fmt.Sprintf("key %s does not exist in secret %s", secretKeyRef.Key, secretKeyRef.Name) 37 | return "", errors.New(errMsg) 38 | } 39 | 40 | return string(secretVal), nil 41 | } 42 | 43 | type OperatorTokenSecretBuilder struct { 44 | client.Object 45 | 46 | Name string 47 | Token string 48 | } 49 | 50 | func (b *OperatorTokenSecretBuilder) Build(obj client.Object) error { 51 | secret, ok := obj.(*corev1.Secret) 52 | if !ok { 53 | return errors.New("failed to cast to Job object") 54 | } 55 | 56 | if secret.ObjectMeta.Name == "" { 57 | secret.ObjectMeta.Name = b.Name 58 | } 59 | secret.ObjectMeta.Namespace = b.GetNamespace() 60 | 61 | secret.Data = map[string][]byte{ 62 | wellKnownNameForOperatorToken: []byte(b.Token), 63 | } 64 | 65 | return nil 66 | } 67 | 68 | func (b *OperatorTokenSecretBuilder) Placeholder(obj client.Object) client.Object { 69 | return &corev1.Secret{ 70 | ObjectMeta: metav1.ObjectMeta{ 71 | Name: b.Name, 72 | Namespace: obj.GetNamespace(), 73 | }, 74 | } 75 | } 76 | 77 | func GetOperatorTokenSecretBuilder(obj client.Object, operatorToken string) ResourceBuilder { 78 | return &OperatorTokenSecretBuilder{ 79 | Object: obj, 80 | 81 | Name: fmt.Sprintf(OperatorTokenSecretNameFormat, obj.GetName()), 82 | Token: operatorToken, 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /internal/resources/security_context.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | corev1 "k8s.io/api/core/v1" 5 | 6 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/ptr" 7 | ) 8 | 9 | func contains(s []corev1.Capability, v corev1.Capability) bool { 10 | for _, vs := range s { 11 | if vs == v { 12 | return true 13 | } 14 | } 15 | return false 16 | } 17 | 18 | func mergeSecurityContextWithDefaults(context *corev1.SecurityContext) *corev1.SecurityContext { 19 | var result *corev1.SecurityContext 20 | defaultCapabilities := []corev1.Capability{"SYS_RAWIO"} 21 | 22 | if context != nil { 23 | result = context.DeepCopy() 24 | } else { 25 | result = &corev1.SecurityContext{} 26 | } 27 | 28 | // set defaults 29 | 30 | if result.Privileged == nil { 31 | result.Privileged = ptr.Bool(false) 32 | } 33 | 34 | if result.Capabilities == nil { 35 | result.Capabilities = &corev1.Capabilities{ 36 | Add: []corev1.Capability{}, 37 | } 38 | } 39 | 40 | for _, defaultCapability := range defaultCapabilities { 41 | if !contains(result.Capabilities.Add, defaultCapability) { 42 | result.Capabilities.Add = append(result.Capabilities.Add, defaultCapability) 43 | } 44 | } 45 | 46 | return result 47 | } 48 | -------------------------------------------------------------------------------- /internal/resources/security_context_test.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "testing" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | 8 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/ptr" 9 | 10 | . "github.com/onsi/ginkgo/v2" 11 | . "github.com/onsi/gomega" 12 | ) 13 | 14 | func TestSecurityContextMerge(t *testing.T) { 15 | RegisterFailHandler(Fail) 16 | RunSpecs(t, "SecurityContext builder") 17 | } 18 | 19 | var _ = Describe("SecurityContext builder", func() { 20 | It("no securityContext passed", func() { 21 | Expect(mergeSecurityContextWithDefaults(nil)).Should(BeEquivalentTo(&corev1.SecurityContext{ 22 | Privileged: ptr.Bool(false), 23 | Capabilities: &corev1.Capabilities{ 24 | Add: []corev1.Capability{"SYS_RAWIO"}, 25 | }, 26 | })) 27 | }) 28 | It("securityContext with Capabilities passed", func() { 29 | ctx := &corev1.SecurityContext{ 30 | Privileged: ptr.Bool(false), 31 | Capabilities: &corev1.Capabilities{ 32 | Add: []corev1.Capability{"SYS_PTRACE"}, 33 | }, 34 | } 35 | Expect(mergeSecurityContextWithDefaults(ctx)).Should(BeEquivalentTo(&corev1.SecurityContext{ 36 | Privileged: ptr.Bool(false), 37 | Capabilities: &corev1.Capabilities{ 38 | Add: []corev1.Capability{"SYS_PTRACE", "SYS_RAWIO"}, 39 | }, 40 | })) 41 | }) 42 | It("securityContext without Capabilities passed", func() { 43 | ctx := &corev1.SecurityContext{ 44 | Privileged: ptr.Bool(true), 45 | RunAsUser: ptr.Int64(10), 46 | } 47 | Expect(mergeSecurityContextWithDefaults(ctx)).Should(BeEquivalentTo(&corev1.SecurityContext{ 48 | Privileged: ptr.Bool(true), 49 | RunAsUser: ptr.Int64(10), 50 | Capabilities: &corev1.Capabilities{ 51 | Add: []corev1.Capability{"SYS_RAWIO"}, 52 | }, 53 | })) 54 | }) 55 | }) 56 | -------------------------------------------------------------------------------- /internal/resources/service.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | corev1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "sigs.k8s.io/controller-runtime/pkg/client" 10 | ) 11 | 12 | const ( 13 | DefaultNameFormat = "%s" 14 | ) 15 | 16 | type ServiceBuilder struct { 17 | client.Object 18 | 19 | NameFormat string 20 | 21 | Ports []corev1.ServicePort 22 | Headless bool 23 | 24 | IPFamilies []corev1.IPFamily 25 | IPFamilyPolicy *corev1.IPFamilyPolicyType 26 | 27 | Labels map[string]string 28 | SelectorLabels map[string]string 29 | 30 | Annotations map[string]string 31 | } 32 | 33 | func (b *ServiceBuilder) Build(obj client.Object) error { 34 | service, ok := obj.(*corev1.Service) 35 | if !ok { 36 | return errors.New("failed to cast to Service object") 37 | } 38 | 39 | if b.NameFormat == "" { 40 | b.NameFormat = DefaultNameFormat 41 | } 42 | 43 | if service.ObjectMeta.Name == "" { 44 | service.ObjectMeta.Name = fmt.Sprintf(b.NameFormat, b.GetName()) 45 | } 46 | service.ObjectMeta.Namespace = b.GetNamespace() 47 | service.ObjectMeta.Labels = b.Labels 48 | service.ObjectMeta.Annotations = b.Annotations 49 | 50 | service.Spec.Ports = b.Ports 51 | service.Spec.Selector = b.SelectorLabels 52 | 53 | if len(b.IPFamilies) > 0 { 54 | service.Spec.IPFamilies = b.IPFamilies 55 | } 56 | 57 | if b.IPFamilyPolicy != nil { 58 | service.Spec.IPFamilyPolicy = b.IPFamilyPolicy 59 | } 60 | 61 | if b.Headless && service.Spec.ClusterIP == "" { 62 | service.Spec.ClusterIP = "None" 63 | } 64 | 65 | for _, port := range service.Spec.Ports { 66 | if port.NodePort > 0 { 67 | service.Spec.Type = corev1.ServiceTypeNodePort 68 | } 69 | } 70 | 71 | return nil 72 | } 73 | 74 | func (b *ServiceBuilder) Placeholder(cr client.Object) client.Object { 75 | if b.NameFormat == "" { 76 | b.NameFormat = "%s" 77 | } 78 | 79 | return &corev1.Service{ 80 | ObjectMeta: metav1.ObjectMeta{ 81 | Name: fmt.Sprintf(b.NameFormat, b.GetName()), 82 | Namespace: cr.GetNamespace(), 83 | }, 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /internal/resources/servicemonitor.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "errors" 5 | 6 | monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/apimachinery/pkg/util/intstr" 9 | "sigs.k8s.io/controller-runtime/pkg/client" 10 | 11 | api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 12 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" 13 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/metrics" 14 | ) 15 | 16 | type ServiceMonitorBuilder struct { 17 | client.Object 18 | 19 | Name string 20 | MetricsServices []metrics.Service 21 | TargetPort int 22 | Options *api.MonitoringOptions 23 | 24 | Labels labels.Labels 25 | SelectorLabels labels.Labels 26 | } 27 | 28 | func (b *ServiceMonitorBuilder) Build(obj client.Object) error { 29 | sm, ok := obj.(*monitoringv1.ServiceMonitor) 30 | if !ok { 31 | return errors.New("failed to cast to ServiceMonitor object") 32 | } 33 | 34 | if sm.ObjectMeta.Name == "" { 35 | sm.ObjectMeta.Name = b.Object.GetName() 36 | } 37 | 38 | sm.ObjectMeta.Namespace = b.GetNamespace() 39 | sm.ObjectMeta.Labels = b.Labels 40 | 41 | sm.Spec.Endpoints = b.buildEndpoints() 42 | sm.Spec.NamespaceSelector = monitoringv1.NamespaceSelector{ 43 | MatchNames: []string{ 44 | b.GetNamespace(), 45 | }, 46 | } 47 | 48 | sm.Spec.Selector = metav1.LabelSelector{ 49 | MatchLabels: b.SelectorLabels, 50 | } 51 | 52 | return nil 53 | } 54 | 55 | func (b *ServiceMonitorBuilder) buildEndpoints() []monitoringv1.Endpoint { 56 | endpoints := make([]monitoringv1.Endpoint, 0, len(b.MetricsServices)) 57 | 58 | for _, service := range b.MetricsServices { 59 | metricRelabelings := service.Relabelings 60 | if len(b.Options.MetricRelabelings) > 0 { 61 | metricRelabelings = append(metricRelabelings, b.Options.MetricRelabelings...) 62 | } 63 | 64 | endpoints = append(endpoints, monitoringv1.Endpoint{ 65 | Path: service.Path, 66 | TargetPort: &intstr.IntOrString{IntVal: int32(b.TargetPort)}, 67 | MetricRelabelConfigs: metricRelabelings, 68 | }) 69 | } 70 | 71 | return endpoints 72 | } 73 | 74 | func (b *ServiceMonitorBuilder) Placeholder(cr client.Object) client.Object { 75 | return &monitoringv1.ServiceMonitor{ 76 | ObjectMeta: metav1.ObjectMeta{ 77 | Name: cr.GetName(), 78 | Namespace: cr.GetNamespace(), 79 | }, 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /internal/resources/storagenodeset.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "errors" 5 | 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | "k8s.io/client-go/rest" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | 10 | api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" 11 | "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" 12 | ) 13 | 14 | type StorageNodeSetBuilder struct { 15 | client.Object 16 | 17 | Name string 18 | Labels map[string]string 19 | Annotations map[string]string 20 | 21 | StorageNodeSetSpec api.StorageNodeSetSpec 22 | } 23 | 24 | type StorageNodeSetResource struct { 25 | *api.StorageNodeSet 26 | } 27 | 28 | func (b *StorageNodeSetBuilder) Build(obj client.Object) error { 29 | sns, ok := obj.(*api.StorageNodeSet) 30 | if !ok { 31 | return errors.New("failed to cast to StorageNodeSet object") 32 | } 33 | 34 | if sns.ObjectMeta.Name == "" { 35 | sns.ObjectMeta.Name = b.Name 36 | } 37 | sns.ObjectMeta.Namespace = b.GetNamespace() 38 | 39 | sns.ObjectMeta.Labels = b.Labels 40 | sns.ObjectMeta.Annotations = b.Annotations 41 | 42 | sns.Spec = b.StorageNodeSetSpec 43 | 44 | return nil 45 | } 46 | 47 | func (b *StorageNodeSetBuilder) Placeholder(cr client.Object) client.Object { 48 | return &api.StorageNodeSet{ 49 | ObjectMeta: metav1.ObjectMeta{ 50 | Name: b.Name, 51 | Namespace: cr.GetNamespace(), 52 | }, 53 | } 54 | } 55 | 56 | func (b *StorageNodeSetResource) GetResourceBuilders(restConfig *rest.Config) []ResourceBuilder { 57 | ydbCr := api.RecastStorageNodeSet(b.Unwrap()) 58 | clusterBuilder := NewCluster(ydbCr) 59 | 60 | statefulSetName := b.Name 61 | statefulSetLabels := clusterBuilder.buildLabels() 62 | statefulSetAnnotations := CopyDict(b.Spec.AdditionalAnnotations) 63 | statefulSetAnnotations[annotations.ConfigurationChecksum] = SHAChecksum(b.Spec.Configuration) 64 | 65 | var resourceBuilders []ResourceBuilder 66 | resourceBuilders = append( 67 | resourceBuilders, 68 | &StorageStatefulSetBuilder{ 69 | Storage: ydbCr, 70 | RestConfig: restConfig, 71 | 72 | Name: statefulSetName, 73 | Labels: statefulSetLabels, 74 | Annotations: statefulSetAnnotations, 75 | }, 76 | ) 77 | 78 | return resourceBuilders 79 | } 80 | 81 | func NewStorageNodeSet(storageNodeSet *api.StorageNodeSet) StorageNodeSetResource { 82 | crStorageNodeSet := storageNodeSet.DeepCopy() 83 | 84 | if crStorageNodeSet.Spec.Service.Status.TLSConfiguration == nil { 85 | crStorageNodeSet.Spec.Service.Status.TLSConfiguration = &api.TLSConfiguration{Enabled: false} 86 | } 87 | 88 | return StorageNodeSetResource{StorageNodeSet: crStorageNodeSet} 89 | } 90 | 91 | func (b *StorageNodeSetResource) Unwrap() *api.StorageNodeSet { 92 | return b.DeepCopy() 93 | } 94 | -------------------------------------------------------------------------------- /samples/database.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: Database 3 | metadata: 4 | name: database-sample 5 | spec: 6 | image: 7 | name: cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17 8 | nodes: 3 9 | resources: 10 | storageUnits: 11 | - count: 1 12 | unitKind: ssd 13 | storageClusterRef: 14 | name: storage-sample 15 | -------------------------------------------------------------------------------- /samples/databasemonitoring.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: DatabaseMonitoring 3 | metadata: 4 | name: database-sample 5 | labels: 6 | release: prom 7 | spec: 8 | databaseRef: 9 | name: database-sample 10 | -------------------------------------------------------------------------------- /samples/kind/database-3dc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: Database 3 | metadata: 4 | name: database-kind-sample 5 | spec: 6 | image: 7 | name: cr.yandex/crptqonuodf51kdj7a7d/ydb:24.2.7 8 | affinity: 9 | podAntiAffinity: 10 | requiredDuringSchedulingIgnoredDuringExecution: 11 | - labelSelector: 12 | matchExpressions: 13 | - key: app.kubernetes.io/component 14 | operator: In 15 | values: 16 | - storage-node 17 | topologyKey: "kubernetes.io/hostname" 18 | nodes: 3 19 | resources: 20 | storageUnits: 21 | - count: 1 22 | unitKind: ssd 23 | storageClusterRef: 24 | name: storage-kind-sample 25 | -------------------------------------------------------------------------------- /samples/kind/database.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: Database 3 | metadata: 4 | name: database-kind-sample 5 | spec: 6 | image: 7 | name: cr.yandex/crptqonuodf51kdj7a7d/ydb:24.2.7 8 | nodes: 1 9 | resources: 10 | storageUnits: 11 | - count: 1 12 | unitKind: ssd 13 | storageClusterRef: 14 | name: storage-kind-sample 15 | -------------------------------------------------------------------------------- /samples/kind/kind-3dc-config.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | nodes: 4 | - role: control-plane 5 | - role: worker 6 | labels: 7 | topology.kubernetes.io/zone: az-1 8 | - role: worker 9 | labels: 10 | topology.kubernetes.io/zone: az-2 11 | - role: worker 12 | labels: 13 | topology.kubernetes.io/zone: az-3 14 | -------------------------------------------------------------------------------- /samples/kind/kind-config.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | nodes: 4 | - role: control-plane 5 | - role: worker 6 | labels: 7 | topology.kubernetes.io/zone: az-1 8 | -------------------------------------------------------------------------------- /samples/kind/storage-mirror-3dc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: Storage 3 | metadata: 4 | name: storage-kind-sample 5 | spec: 6 | image: 7 | name: cr.yandex/crptqonuodf51kdj7a7d/ydb:24.2.7 8 | affinity: 9 | podAntiAffinity: 10 | requiredDuringSchedulingIgnoredDuringExecution: 11 | - labelSelector: 12 | matchExpressions: 13 | - key: app.kubernetes.io/component 14 | operator: In 15 | values: 16 | - storage-node 17 | topologyKey: "kubernetes.io/hostname" 18 | dataStore: [] 19 | nodes: 3 20 | domain: Root 21 | erasure: mirror-3-dc 22 | configuration: |- 23 | actor_system_config: 24 | cpu_count: 1 25 | node_type: STORAGE 26 | use_auto_config: true 27 | blob_storage_config: 28 | service_set: 29 | groups: 30 | - erasure_species: mirror-3-dc 31 | rings: 32 | - fail_domains: 33 | - vdisk_locations: 34 | - node_id: 1 35 | pdisk_category: SSD 36 | path: SectorMap:1:64 37 | - vdisk_locations: 38 | - node_id: 1 39 | pdisk_category: SSD 40 | path: SectorMap:2:64 41 | - vdisk_locations: 42 | - node_id: 1 43 | pdisk_category: SSD 44 | path: SectorMap:3:64 45 | - fail_domains: 46 | - vdisk_locations: 47 | - node_id: 2 48 | pdisk_category: SSD 49 | path: SectorMap:1:64 50 | - vdisk_locations: 51 | - node_id: 2 52 | pdisk_category: SSD 53 | path: SectorMap:2:64 54 | - vdisk_locations: 55 | - node_id: 2 56 | pdisk_category: SSD 57 | path: SectorMap:3:64 58 | - fail_domains: 59 | - vdisk_locations: 60 | - node_id: 3 61 | pdisk_category: SSD 62 | path: SectorMap:1:64 63 | - vdisk_locations: 64 | - node_id: 3 65 | pdisk_category: SSD 66 | path: SectorMap:2:64 67 | - vdisk_locations: 68 | - node_id: 3 69 | pdisk_category: SSD 70 | path: SectorMap:3:64 71 | channel_profile_config: 72 | profile: 73 | - channel: 74 | - erasure_species: mirror-3-dc 75 | pdisk_category: 0 76 | storage_pool_kind: ssd 77 | - erasure_species: mirror-3-dc 78 | pdisk_category: 0 79 | storage_pool_kind: ssd 80 | - erasure_species: mirror-3-dc 81 | pdisk_category: 0 82 | storage_pool_kind: ssd 83 | profile_id: 0 84 | domains_config: 85 | domain: 86 | - name: Root 87 | storage_pool_types: 88 | - kind: ssd 89 | pool_config: 90 | box_id: 1 91 | erasure_species: mirror-3-dc 92 | kind: ssd 93 | pdisk_filter: 94 | - property: 95 | - type: SSD 96 | vdisk_kind: Default 97 | state_storage: 98 | - ring: 99 | node: [1, 2, 3] 100 | nto_select: 3 101 | ssid: 1 102 | grpc_config: 103 | port: 2135 104 | host_configs: 105 | - drive: 106 | - path: SectorMap:1:64 107 | type: SSD 108 | - path: SectorMap:2:64 109 | type: SSD 110 | - path: SectorMap:3:64 111 | type: SSD 112 | host_config_id: 1 113 | static_erasure: mirror-3-dc 114 | -------------------------------------------------------------------------------- /samples/kind/storage.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: Storage 3 | metadata: 4 | name: storage-kind-sample 5 | spec: 6 | dataStore: [] 7 | image: 8 | name: cr.yandex/crptqonuodf51kdj7a7d/ydb:24.2.7 9 | nodes: 1 10 | domain: Root 11 | erasure: none 12 | configuration: |- 13 | actor_system_config: 14 | cpu_count: 1 15 | node_type: STORAGE 16 | use_auto_config: true 17 | blob_storage_config: 18 | service_set: 19 | groups: 20 | - erasure_species: none 21 | rings: 22 | - fail_domains: 23 | - vdisk_locations: 24 | - node_id: 1 25 | path: SectorMap:1:64 26 | pdisk_category: SSD 27 | channel_profile_config: 28 | profile: 29 | - channel: 30 | - erasure_species: none 31 | pdisk_category: 0 32 | storage_pool_kind: ssd 33 | - erasure_species: none 34 | pdisk_category: 0 35 | storage_pool_kind: ssd 36 | - erasure_species: none 37 | pdisk_category: 0 38 | storage_pool_kind: ssd 39 | profile_id: 0 40 | domains_config: 41 | domain: 42 | - name: Root 43 | storage_pool_types: 44 | - kind: ssd 45 | pool_config: 46 | box_id: 1 47 | erasure_species: none 48 | kind: ssd 49 | pdisk_filter: 50 | - property: 51 | - type: SSD 52 | vdisk_kind: Default 53 | state_storage: 54 | - ring: 55 | node: [1] 56 | nto_select: 1 57 | ssid: 1 58 | grpc_config: 59 | port: 2135 60 | host_configs: 61 | - drive: 62 | - path: SectorMap:1:64 63 | type: SSD 64 | host_config_id: 1 65 | static_erasure: none 66 | -------------------------------------------------------------------------------- /samples/minikube/database.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: Database 3 | metadata: 4 | name: database-minikube-sample 5 | spec: 6 | image: 7 | name: cr.yandex/crptqonuodf51kdj7a7d/ydb:24.2.7 8 | nodes: 1 9 | domain: Root 10 | service: 11 | grpc: 12 | externalHost: localhost 13 | resources: 14 | storageUnits: 15 | - count: 1 16 | unitKind: ssd 17 | storageClusterRef: 18 | name: storage-minikube-sample 19 | -------------------------------------------------------------------------------- /samples/minikube/storage.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: Storage 3 | metadata: 4 | name: storage-minikube-sample 5 | spec: 6 | dataStore: [] 7 | image: 8 | name: cr.yandex/crptqonuodf51kdj7a7d/ydb:24.2.7 9 | nodes: 1 10 | domain: Root 11 | erasure: none 12 | configuration: |- 13 | static_erasure: none 14 | host_configs: 15 | - drive: 16 | - path: SectorMap:1:64 17 | type: SSD 18 | host_config_id: 1 19 | grpc_config: 20 | port: 2135 21 | domains_config: 22 | domain: 23 | - name: Root 24 | storage_pool_types: 25 | - kind: ssd 26 | pool_config: 27 | box_id: 1 28 | erasure_species: none 29 | kind: ssd 30 | pdisk_filter: 31 | - property: 32 | - type: SSD 33 | vdisk_kind: Default 34 | state_storage: 35 | - ring: 36 | node: 37 | - 1 38 | nto_select: 1 39 | ssid: 1 40 | actor_system_config: 41 | executor: 42 | - name: System 43 | spin_threshold: 0 44 | threads: 2 45 | type: BASIC 46 | - name: User 47 | spin_threshold: 0 48 | threads: 3 49 | type: BASIC 50 | - name: Batch 51 | spin_threshold: 0 52 | threads: 2 53 | type: BASIC 54 | - name: IO 55 | threads: 1 56 | time_per_mailbox_micro_secs: 100 57 | type: IO 58 | - name: IC 59 | spin_threshold: 10 60 | threads: 1 61 | time_per_mailbox_micro_secs: 100 62 | type: BASIC 63 | scheduler: 64 | progress_threshold: 10000 65 | resolution: 256 66 | spin_threshold: 0 67 | blob_storage_config: 68 | service_set: 69 | groups: 70 | - erasure_species: none 71 | rings: 72 | - fail_domains: 73 | - vdisk_locations: 74 | - node_id: 1 75 | path: SectorMap:1:64 76 | pdisk_category: SSD 77 | channel_profile_config: 78 | profile: 79 | - channel: 80 | - erasure_species: none 81 | pdisk_category: 1 82 | storage_pool_kind: ssd 83 | - erasure_species: none 84 | pdisk_category: 1 85 | storage_pool_kind: ssd 86 | - erasure_species: none 87 | pdisk_category: 1 88 | storage_pool_kind: ssd 89 | profile_id: 0 90 | -------------------------------------------------------------------------------- /samples/remote-rbac.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: yc-dev 6 | namespace: ydb 7 | --- 8 | apiVersion: v1 9 | kind: Secret 10 | metadata: 11 | name: yc-dev-token 12 | namespace: ydb 13 | annotations: 14 | kubernetes.io/service-account.name: yc-dev 15 | type: kubernetes.io/service-account-token 16 | --- 17 | apiVersion: rbac.authorization.k8s.io/v1 18 | kind: ClusterRole 19 | metadata: 20 | name: ydb-operator-remote 21 | namespace: ydb 22 | rules: 23 | - apiGroups: 24 | - "" 25 | resources: 26 | - configmaps 27 | verbs: 28 | - get 29 | - list 30 | - watch 31 | - apiGroups: 32 | - "" 33 | resources: 34 | - secrets 35 | verbs: 36 | - get 37 | - list 38 | - watch 39 | - apiGroups: 40 | - "" 41 | resources: 42 | - services 43 | verbs: 44 | - get 45 | - list 46 | - watch 47 | - apiGroups: 48 | - ydb.tech 49 | resources: 50 | - remotedatabasenodesets 51 | - remotestoragenodesets 52 | verbs: 53 | - get 54 | - list 55 | - watch 56 | - apiGroups: 57 | - ydb.tech 58 | resources: 59 | - remotedatabasenodesets 60 | - remotestoragenodesets 61 | verbs: 62 | - update 63 | - apiGroups: 64 | - ydb.tech 65 | resources: 66 | - remotedatabasenodesets/status 67 | - remotestoragenodesets/status 68 | verbs: 69 | - get 70 | - patch 71 | - update 72 | - apiGroups: 73 | - "" 74 | resources: 75 | - events 76 | verbs: 77 | - create 78 | - patch 79 | --- 80 | apiVersion: rbac.authorization.k8s.io/v1 81 | kind: ClusterRoleBinding 82 | metadata: 83 | name: ydb-operator-remote-rolebinding 84 | namespace: ydb 85 | roleRef: 86 | apiGroup: rbac.authorization.k8s.io 87 | kind: ClusterRole 88 | name: ydb-operator-remote 89 | subjects: 90 | - kind: ServiceAccount 91 | name: yc-dev 92 | namespace: ydb 93 | -------------------------------------------------------------------------------- /samples/storage-block-4-2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: Storage 3 | metadata: 4 | name: storage-sample 5 | spec: 6 | dataStore: 7 | - accessModes: 8 | - ReadWriteOnce 9 | resources: 10 | requests: 11 | storage: 93Gi 12 | volumeMode: Block 13 | image: 14 | name: cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17 15 | nodes: 8 16 | erasure: block-4-2 17 | configuration: |- 18 | static_erasure: block-4-2 19 | host_configs: 20 | - drive: 21 | - path: /dev/kikimr_ssd_00 22 | type: SSD 23 | host_config_id: 1 24 | grpc_config: 25 | port: 2135 26 | domains_config: 27 | domain: 28 | - name: Root 29 | storage_pool_types: 30 | - kind: ssd 31 | pool_config: 32 | box_id: 1 33 | erasure_species: block-4-2 34 | kind: ssd 35 | pdisk_filter: 36 | - property: 37 | - type: SSD 38 | vdisk_kind: Default 39 | state_storage: 40 | - ring: 41 | node: [ 1, 2, 3, 4, 5, 6, 7, 8 ] 42 | nto_select: 5 43 | ssid: 1 44 | actor_system_config: 45 | batch_executor: 2 46 | io_executor: 3 47 | executor: 48 | - name: System 49 | spin_threshold: 0 50 | threads: 2 51 | type: BASIC 52 | - name: User 53 | spin_threshold: 0 54 | threads: 3 55 | type: BASIC 56 | - name: Batch 57 | spin_threshold: 0 58 | threads: 2 59 | type: BASIC 60 | - name: IO 61 | threads: 1 62 | time_per_mailbox_micro_secs: 100 63 | type: IO 64 | - name: IC 65 | spin_threshold: 10 66 | threads: 1 67 | time_per_mailbox_micro_secs: 100 68 | type: BASIC 69 | scheduler: 70 | progress_threshold: 10000 71 | resolution: 256 72 | spin_threshold: 0 73 | service_executor: 74 | - executor_id: 4 75 | service_name: Interconnect 76 | blob_storage_config: 77 | service_set: 78 | availability_domains: 1 79 | groups: 80 | - erasure_species: block-4-2 81 | group_id: 0 82 | group_generation: 1 83 | rings: 84 | - fail_domains: 85 | - vdisk_locations: 86 | - node_id: 1 87 | pdisk_category: SSD 88 | path: /dev/kikimr_ssd_00 89 | - vdisk_locations: 90 | - node_id: 2 91 | pdisk_category: SSD 92 | path: /dev/kikimr_ssd_00 93 | - vdisk_locations: 94 | - node_id: 3 95 | pdisk_category: SSD 96 | path: /dev/kikimr_ssd_00 97 | - vdisk_locations: 98 | - node_id: 4 99 | pdisk_category: SSD 100 | path: /dev/kikimr_ssd_00 101 | - vdisk_locations: 102 | - node_id: 5 103 | pdisk_category: SSD 104 | path: /dev/kikimr_ssd_00 105 | - vdisk_locations: 106 | - node_id: 6 107 | pdisk_category: SSD 108 | path: /dev/kikimr_ssd_00 109 | - vdisk_locations: 110 | - node_id: 7 111 | pdisk_category: SSD 112 | path: /dev/kikimr_ssd_00 113 | - vdisk_locations: 114 | - node_id: 8 115 | pdisk_category: SSD 116 | path: /dev/kikimr_ssd_00 117 | channel_profile_config: 118 | profile: 119 | - channel: 120 | - erasure_species: block-4-2 121 | pdisk_category: 1 122 | storage_pool_kind: ssd 123 | - erasure_species: block-4-2 124 | pdisk_category: 1 125 | storage_pool_kind: ssd 126 | - erasure_species: block-4-2 127 | pdisk_category: 1 128 | storage_pool_kind: ssd 129 | profile_id: 0 130 | -------------------------------------------------------------------------------- /samples/storage-mirror-3dc-nodeset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: Storage 3 | metadata: 4 | name: storage-sample 5 | spec: 6 | dataStore: 7 | - volumeMode: Block 8 | accessModes: 9 | - ReadWriteOnce 10 | resources: 11 | requests: 12 | storage: 80Gi 13 | image: 14 | name: cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17 15 | nodes: 9 16 | nodeSets: 17 | - name: nodeset-1 18 | nodes: 9 19 | remote: 20 | cluster: "yc-dev" 21 | erasure: mirror-3-dc 22 | configuration: |- 23 | static_erasure: mirror-3-dc 24 | host_configs: 25 | - drive: 26 | - path: /dev/kikimr_ssd_00 27 | type: SSD 28 | host_config_id: 1 29 | grpc_config: 30 | port: 2135 31 | domains_config: 32 | domain: 33 | - name: Root 34 | storage_pool_types: 35 | - kind: ssd 36 | pool_config: 37 | box_id: 1 38 | erasure_species: mirror-3-dc 39 | kind: ssd 40 | pdisk_filter: 41 | - property: 42 | - type: SSD 43 | vdisk_kind: Default 44 | state_storage: 45 | - ring: 46 | node: [ 1, 2, 3, 4, 5, 6, 7, 8, 9 ] 47 | nto_select: 5 48 | ssid: 1 49 | actor_system_config: 50 | batch_executor: 2 51 | io_executor: 3 52 | executor: 53 | - name: System 54 | spin_threshold: 0 55 | threads: 2 56 | type: BASIC 57 | - name: User 58 | spin_threshold: 0 59 | threads: 3 60 | type: BASIC 61 | - name: Batch 62 | spin_threshold: 0 63 | threads: 2 64 | type: BASIC 65 | - name: IO 66 | threads: 1 67 | time_per_mailbox_micro_secs: 100 68 | type: IO 69 | - name: IC 70 | spin_threshold: 10 71 | threads: 1 72 | time_per_mailbox_micro_secs: 100 73 | type: BASIC 74 | scheduler: 75 | progress_threshold: 10000 76 | resolution: 256 77 | spin_threshold: 0 78 | service_executor: 79 | - executor_id: 4 80 | service_name: Interconnect 81 | blob_storage_config: 82 | service_set: 83 | availability_domains: 1 84 | groups: 85 | - erasure_species: mirror-3-dc 86 | group_id: 0 87 | group_generation: 1 88 | rings: 89 | - fail_domains: 90 | - vdisk_locations: 91 | - node_id: 1 92 | pdisk_category: SSD 93 | path: /dev/kikimr_ssd_00 94 | - vdisk_locations: 95 | - node_id: 2 96 | pdisk_category: SSD 97 | path: /dev/kikimr_ssd_00 98 | - vdisk_locations: 99 | - node_id: 3 100 | pdisk_category: SSD 101 | path: /dev/kikimr_ssd_00 102 | - fail_domains: 103 | - vdisk_locations: 104 | - node_id: 4 105 | pdisk_category: SSD 106 | path: /dev/kikimr_ssd_00 107 | - vdisk_locations: 108 | - node_id: 5 109 | pdisk_category: SSD 110 | path: /dev/kikimr_ssd_00 111 | - vdisk_locations: 112 | - node_id: 6 113 | pdisk_category: SSD 114 | path: /dev/kikimr_ssd_00 115 | - fail_domains: 116 | - vdisk_locations: 117 | - node_id: 7 118 | pdisk_category: SSD 119 | path: /dev/kikimr_ssd_00 120 | - vdisk_locations: 121 | - node_id: 8 122 | pdisk_category: SSD 123 | path: /dev/kikimr_ssd_00 124 | - vdisk_locations: 125 | - node_id: 9 126 | pdisk_category: SSD 127 | path: /dev/kikimr_ssd_00 128 | channel_profile_config: 129 | profile: 130 | - channel: 131 | - erasure_species: mirror-3-dc 132 | pdisk_category: 1 133 | storage_pool_kind: ssd 134 | - erasure_species: mirror-3-dc 135 | pdisk_category: 1 136 | storage_pool_kind: ssd 137 | - erasure_species: mirror-3-dc 138 | pdisk_category: 1 139 | storage_pool_kind: ssd 140 | profile_id: 0 141 | -------------------------------------------------------------------------------- /samples/storage-mirror-3dc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: Storage 3 | metadata: 4 | name: storage-sample 5 | spec: 6 | dataStore: 7 | - volumeMode: Block 8 | accessModes: 9 | - ReadWriteOnce 10 | resources: 11 | requests: 12 | storage: 80Gi 13 | image: 14 | name: cr.yandex/crptqonuodf51kdj7a7d/ydb:24.2.7 15 | nodes: 9 16 | erasure: mirror-3-dc 17 | configuration: |- 18 | static_erasure: mirror-3-dc 19 | host_configs: 20 | - drive: 21 | - path: /dev/kikimr_ssd_00 22 | type: SSD 23 | host_config_id: 1 24 | grpc_config: 25 | port: 2135 26 | domains_config: 27 | domain: 28 | - name: Root 29 | storage_pool_types: 30 | - kind: ssd 31 | pool_config: 32 | box_id: 1 33 | erasure_species: mirror-3-dc 34 | kind: ssd 35 | pdisk_filter: 36 | - property: 37 | - type: SSD 38 | vdisk_kind: Default 39 | state_storage: 40 | - ring: 41 | node: [ 1, 2, 3, 4, 5, 6, 7, 8, 9 ] 42 | nto_select: 5 43 | ssid: 1 44 | actor_system_config: 45 | batch_executor: 2 46 | io_executor: 3 47 | executor: 48 | - name: System 49 | spin_threshold: 0 50 | threads: 2 51 | type: BASIC 52 | - name: User 53 | spin_threshold: 0 54 | threads: 3 55 | type: BASIC 56 | - name: Batch 57 | spin_threshold: 0 58 | threads: 2 59 | type: BASIC 60 | - name: IO 61 | threads: 1 62 | time_per_mailbox_micro_secs: 100 63 | type: IO 64 | - name: IC 65 | spin_threshold: 10 66 | threads: 1 67 | time_per_mailbox_micro_secs: 100 68 | type: BASIC 69 | scheduler: 70 | progress_threshold: 10000 71 | resolution: 256 72 | spin_threshold: 0 73 | service_executor: 74 | - executor_id: 4 75 | service_name: Interconnect 76 | blob_storage_config: 77 | service_set: 78 | availability_domains: 1 79 | groups: 80 | - erasure_species: mirror-3-dc 81 | group_id: 0 82 | group_generation: 1 83 | rings: 84 | - fail_domains: 85 | - vdisk_locations: 86 | - node_id: 1 87 | pdisk_category: SSD 88 | path: /dev/kikimr_ssd_00 89 | - vdisk_locations: 90 | - node_id: 2 91 | pdisk_category: SSD 92 | path: /dev/kikimr_ssd_00 93 | - vdisk_locations: 94 | - node_id: 3 95 | pdisk_category: SSD 96 | path: /dev/kikimr_ssd_00 97 | - fail_domains: 98 | - vdisk_locations: 99 | - node_id: 4 100 | pdisk_category: SSD 101 | path: /dev/kikimr_ssd_00 102 | - vdisk_locations: 103 | - node_id: 5 104 | pdisk_category: SSD 105 | path: /dev/kikimr_ssd_00 106 | - vdisk_locations: 107 | - node_id: 6 108 | pdisk_category: SSD 109 | path: /dev/kikimr_ssd_00 110 | - fail_domains: 111 | - vdisk_locations: 112 | - node_id: 7 113 | pdisk_category: SSD 114 | path: /dev/kikimr_ssd_00 115 | - vdisk_locations: 116 | - node_id: 8 117 | pdisk_category: SSD 118 | path: /dev/kikimr_ssd_00 119 | - vdisk_locations: 120 | - node_id: 9 121 | pdisk_category: SSD 122 | path: /dev/kikimr_ssd_00 123 | channel_profile_config: 124 | profile: 125 | - channel: 126 | - erasure_species: mirror-3-dc 127 | pdisk_category: 1 128 | storage_pool_kind: ssd 129 | - erasure_species: mirror-3-dc 130 | pdisk_category: 1 131 | storage_pool_kind: ssd 132 | - erasure_species: mirror-3-dc 133 | pdisk_category: 1 134 | storage_pool_kind: ssd 135 | profile_id: 0 136 | -------------------------------------------------------------------------------- /samples/storagemonitoring.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: ydb.tech/v1alpha1 2 | kind: StorageMonitoring 3 | metadata: 4 | name: storage-sample 5 | labels: 6 | release: prom 7 | spec: 8 | storageRef: 9 | name: storage-sample 10 | -------------------------------------------------------------------------------- /tests/cfg/kind-cluster-config.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | nodes: 4 | - role: control-plane 5 | 6 | - role: worker 7 | extraPortMappings: 8 | - containerPort: 30001 9 | hostPort: 30001 10 | listenAddress: "127.0.0.1" 11 | protocol: tcp 12 | labels: 13 | topology.kubernetes.io/zone: az-1 14 | worker: true 15 | 16 | - role: worker 17 | labels: 18 | topology.kubernetes.io/zone: az-2 19 | worker: true 20 | 21 | - role: worker 22 | labels: 23 | topology.kubernetes.io/zone: az-3 24 | worker: true 25 | -------------------------------------------------------------------------------- /tests/cfg/operator-local-values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | pullPolicy: IfNotPresent 3 | repository: kind/ydb-operator 4 | tag: current 5 | 6 | webhook: 7 | enabled: true 8 | 9 | patch: 10 | enabled: true 11 | -------------------------------------------------------------------------------- /tests/cfg/operator-values.yaml: -------------------------------------------------------------------------------- 1 | webhook: 2 | enabled: true 3 | 4 | patch: 5 | enabled: true 6 | -------------------------------------------------------------------------------- /tests/data/database.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDhDCCAmygAwIBAgIUUQQsk4wdGfrawpygX64aFtR6/1IwDQYJKoZIhvcNAQEL 3 | BQAwFzEVMBMGA1UEAwwMdGVzdC1yb290LWNhMB4XDTI0MTIwNTEzMjEwMVoXDTM5 4 | MTIwMjEzMjEwMVowDzENMAsGA1UECgwEdGVzdDCCASIwDQYJKoZIhvcNAQEBBQAD 5 | ggEPADCCAQoCggEBAMU+EoeP6971G2uFgo2Sm2Ela8rjZSmivZZ3Xg//rj9gcfle 6 | KNoJ5EAHdwyTfapLoSVvgy1QLun15ibWeRgbBuUt2+DiHnEpaeriUqmktki9UIzl 7 | pAjytDwCmsjbOoLXRhCIa02tkU6rF8JjpwitZnwhTXjJTAkuJiuNvN2EEdacTlx1 8 | ZPdcHQveJTVJy4eOoSA8yc72XG9CWPY8mhLMTOzoZqbRX7MRoZoyYaV8TNAyQmh4 9 | tX045h4u1ZmMkWC06z2n+8Le3wTpu6mccOhS2ETw0j3Jefx78Zafc2s+jX0lCP71 10 | qiQXeEx1vuYQ5+nop2wh2nTFeFrjH+zZ4eyKduUCAwEAAaOBzzCBzDBdBgNVHREE 11 | VjBUgiNkYXRhYmFzZS1ncnBjLnlkYi5zdmMuY2x1c3Rlci5sb2NhbIItKi5kYXRh 12 | YmFzZS1pbnRlcmNvbm5lY3QueWRiLnN2Yy5jbHVzdGVyLmxvY2FsMB0GA1UdJQQW 13 | MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBRe 14 | hFiyRdaUYUiw6KnHiuTcqwh0CjAfBgNVHSMEGDAWgBRPTY3GM3OCesbsHOf9zDIu 15 | T6ubJTANBgkqhkiG9w0BAQsFAAOCAQEAkYb1N40MGhxw07vVDPHrBfuMSgPqSqef 16 | myPtwAIuwPIOILIAIek0yUogeMKF7kv/C5fyRnac3iHz59M7V4PetW7YhLB6G20n 17 | bOpvq1Bp8Lw7WwRviULWHHIsS9OZlekvikEs3jS9H7XZGgmKC4mN3GbCZkpUvRjU 18 | BBsdyKkQsDupofrzbFPaWfgRjUPGuQ27vUrZkPlmQPrZmowJpTIYwMyJxL8qFtip 19 | JWX8qsKRle58L/K64Nx7AbW2LFjey8txJtkkROwpy9Zt7Dn0kvLcjZC2H8Nqdx8o 20 | bPJqXdMlbGEUFDo1W6W/6zYCRUuDVvtM26Yua5DOm+6wJW+sSqlv4Q== 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /tests/data/database.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDFPhKHj+ve9Rtr 3 | hYKNkpthJWvK42Upor2Wd14P/64/YHH5XijaCeRAB3cMk32qS6Elb4MtUC7p9eYm 4 | 1nkYGwblLdvg4h5xKWnq4lKppLZIvVCM5aQI8rQ8AprI2zqC10YQiGtNrZFOqxfC 5 | Y6cIrWZ8IU14yUwJLiYrjbzdhBHWnE5cdWT3XB0L3iU1ScuHjqEgPMnO9lxvQlj2 6 | PJoSzEzs6Gam0V+zEaGaMmGlfEzQMkJoeLV9OOYeLtWZjJFgtOs9p/vC3t8E6bup 7 | nHDoUthE8NI9yXn8e/GWn3NrPo19JQj+9aokF3hMdb7mEOfp6KdsIdp0xXha4x/s 8 | 2eHsinblAgMBAAECggEALgJQ8bDDcTZlD0NtJOd8GaDQQFsjO59T0I+nEB3Q0EVH 9 | yMarSlMQ3FOxdCxKXak3HYOhynXf/6Clr00LobEaPmbgWZh9R+HEbG8fH6XFhHmu 10 | mrMtfI3at33XC7/BqggbtpsPxqaUVNCpoeU7bxV9qLpe9ywjcafDbRjqo5RdUd0q 11 | J29OtBD/tfEFU17Bv0VlYW/IXmGhJp686ZDvUpybrc2qGiWJo5wnGgUpL0OLNk0L 12 | piK7TThjDzBNLCSzq6DOZOwIoNBmGINLK1Q3SjC77zWH3YzZv/u3EjE3v61phQlj 13 | hy2tR3yimFYGxX6ZJduockJgOC4WJznkR2G85HjggQKBgQD5+JB1o5rfo7J7Txgl 14 | afL7EL2v/+VZ7DSs7Zh+zlIbGGr2vNnW5SgOOfpnjAY+8E91D3BzfmWdIb7/BGwB 15 | 6VAq6aH4a1xhOfWRmUgHL1vDfMfxO8hFN2Ixo2QCt+mQv09lZrExpr1RmlHPPija 16 | XWJ0yE85cJHZlfywLQrG85F+gQKBgQDJ//ArSt4idZ4uRyslMj2ntnsBaD8DwVzl 17 | jie1+ohMW26Fsw3059JplmmiPAvQXFzl3oZcOpJjxjHdGP7al3mdzGHbBSLDtW/h 18 | bREUX78RZm9WHpc2K8ZkPxp36EAysZpmKCdkWH7lB/pt7926BUyqhHXk7thrkTeU 19 | PylzE/iOZQKBgE4oHKrbg5IHMcgCO+9+x/0eB+EepoxOIU4sX7DOO7fDE7af55Cc 20 | R8Di+dskWdOV+ZIFSMijrYvKwFgl/ss+MtWoBP+SOekgYRqsDWxJr2xY+H8BjSWv 21 | ImGYz61V6Y5bcqymxiJbGviHwqqEqetUpXMUKkkwXDnm/oHrI2J/R2+BAoGAYlzv 22 | 7ZTqcGtH2I8tUlKRtV5lrXy+2qxI+Tts2O+jeVM4kYBsdmqAiowE6kxFEHQ5hHIE 23 | iVq4OD+lvl1SlM0YGqAQsp9gm155mZMLsxkgqG9yHcSNq4JLfDtCP0toH4degQpi 24 | jDmPqSVmbCxWkyPLfmk8I3uvBUpUfyr2myQJcAUCgYEA+b+ovH8gh3PGfuXtj4zW 25 | 6IjGXnmt4u2YUssF9sBklbTq9Ev8M4h58TlNh1oHWQ6yyXnpsP8vZMFU0iMYybvi 26 | WGfPMtigyiLASjTV7Ws60uQlZ8raHqtb7QN5wJrvGqVxJe6aw/gQmxY8ejNjjWyM 27 | 1QkHQGyLaWJWFUy3blpUBsA= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /tests/data/generate-crts/README.md: -------------------------------------------------------------------------------- 1 | ## Certificates for testing 2 | 3 | `ca.crt` and `ca.key` are just a self-signed CA created like this: 4 | 5 | 6 | ``` 7 | openssl genrsa -out ca.key 2048 8 | openssl req -x509 -new -nodes -key ca.key -sha256 -days 3650 -out ca.crt -subj "/CN=test-root-ca" 9 | ``` 10 | 11 | Then use `generate-test-certs.sh` to generate storage and database certs, which then are used in tests. 12 | 13 | `ca.srl` is a file created by the script as well. 14 | -------------------------------------------------------------------------------- /tests/data/generate-crts/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDDzCCAfegAwIBAgIUPG5Ffwh8I/zAtqSePmlBTGsP2eEwDQYJKoZIhvcNAQEL 3 | BQAwFzEVMBMGA1UEAwwMdGVzdC1yb290LWNhMB4XDTI0MTIwNTExMjk1MFoXDTM0 4 | MTIwMzExMjk1MFowFzEVMBMGA1UEAwwMdGVzdC1yb290LWNhMIIBIjANBgkqhkiG 5 | 9w0BAQEFAAOCAQ8AMIIBCgKCAQEA45Y8h+mKX0//0H6B+KUcmhyis2dlfI8MlQNo 6 | 1qRpsQQKkqY+n6J8mzFPO+XOC/kLia6SShgpGZF79xhC9+Iq+2ARulIbPH3PiUdf 7 | gwnLD/wfFgCmPaFFfJ93v9AY+eWeq00IKkRVp2gfb159C9BZQmoiyPCPOlWuLN/B 8 | ZPMFHZUWPbL+4mvy4BBrcS/+FncUf7dA5ND7lb26G/sXUGWpYPLclhNnu7Hvapi4 9 | pIx60d8Z3+5eOVHEVECqgIU8wUqTrUbg1YMUHSZxdnsIPnL985sa7a66x/GAgMAi 10 | xuAhUBMyxTUXOXqW+GWIlrmOHmiYRp7ARA3dPbYJJ1kdDfJRswIDAQABo1MwUTAd 11 | BgNVHQ4EFgQUT02NxjNzgnrG7Bzn/cwyLk+rmyUwHwYDVR0jBBgwFoAUT02NxjNz 12 | gnrG7Bzn/cwyLk+rmyUwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC 13 | AQEAfpqusEPmGoL/Hzkui/xs20k+JKFQ90e5iZPQLyuES7BKQp1iajMOytAIXlhY 14 | dtt4oSOYmBfl2bs8OU0U2mjGetx0AHWINY7bNzg7wxd4H46iiCitC4qlUlGG23bF 15 | GVQt7/SddmwKoOJBaasnRTBPqVTlreqAF4Ni8bY0kqO3GK5QWZ2sL+Btn9dML8aK 16 | wLb6sW0h7rAjik0l3NcsrKE8UoViWjAgB3Oe9L00GSXaMnfD6V65XnzkXvLOpCdd 17 | wjZHPoWinhqM8ZHm/iFSe2UcL1KG0rdrMg8oBY6zMNrgENEdhqEXNJJrioT9bzKE 18 | FyNQDOxpeql/GJl2MGxj0FXy+Q== 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /tests/data/generate-crts/ca.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDjljyH6YpfT//Q 3 | foH4pRyaHKKzZ2V8jwyVA2jWpGmxBAqSpj6fonybMU875c4L+QuJrpJKGCkZkXv3 4 | GEL34ir7YBG6Uhs8fc+JR1+DCcsP/B8WAKY9oUV8n3e/0Bj55Z6rTQgqRFWnaB9v 5 | Xn0L0FlCaiLI8I86Va4s38Fk8wUdlRY9sv7ia/LgEGtxL/4WdxR/t0Dk0PuVvbob 6 | +xdQZalg8tyWE2e7se9qmLikjHrR3xnf7l45UcRUQKqAhTzBSpOtRuDVgxQdJnF2 7 | ewg+cv3zmxrtrrrH8YCAwCLG4CFQEzLFNRc5epb4ZYiWuY4eaJhGnsBEDd09tgkn 8 | WR0N8lGzAgMBAAECggEAB/+oZ6EeSzSalGSog2lPf13GR3E42znZLVuN/GKRkbZx 9 | +1l7Fffgp+usZznaa11ODywNhCvOi1GA/obhw6iKmNnK2wuLstfmdWKxc/+MyCZf 10 | nqcDMLim9+GllHOR4nve1GfEA7LxzQ0XHb4/vYHjFox4ZdYz5870bNX9triRKMrw 11 | Ru6FMbdcMf8ClTHtqHIQgARRTOFvizqVFElFgrIo4eh8svseed3+xwX98oeC+O0u 12 | WrLW42RpVWIoai+V6OdYzO+uALr8z0IC3yzw1pqLMVt+SLY0nS/qz8b3eruU4BmL 13 | 6i0BQvJF8QFCE+gmtZJSQyCXzOw5jcvlDAm10v/1QQKBgQD+eg13guElgrjvkjT6 14 | a1ry3hA5LfQ7QyqigydyZlkbPfBcnv0dsWXbL+toyu0RqMwcKBGG3/7bVpGx/0gb 15 | EgtnDWgS/PyWpuW+scwhqryzCKLnbUCor3GIo7CkM+71m5BcOHNTUPoqrMt6/W6a 16 | mQ4z+VNpVQlRrx3YXp2zbO/dgwKBgQDk8vqmioezNNNhnk/7LhFITN7TavsVT3LT 17 | jgs9CTSwEKFLC8eo/sKPaTKTcPxhxrt9eOWhYbNNQ9WYzy/ulA/vKvfkZjvtUaHl 18 | sC35E9FcMW6lSiM8LOQlPnJYq6VqIQOdTgtBp1lsmQvkRbAF7Wq2Fy8BjiwPJfV1 19 | CJejygI0EQKBgCZs96ucL7MiUhqa0TUfENSrg3ee4MoyEjYH5+T2X24lpC3YNBBP 20 | wTmfusRQIAwSmP+HbV4YZLtqDwX5rkGoL+CXvadgXCPDf92Tq2dKCMRgAXlAngra 21 | syIW1Y116hdcLig+vetOxve6r98adaESi3p9o4K8PHQBJViOsPFu+alRAoGAUtb8 22 | DIB5Y0VM6rheljLv++ocggDmgqpxkMyHknkfQEl0IvRLNQGhIkTdEO5D05kVw+uX 23 | otH4D4/o3FazMC8QqOgyM8kuC8uKudIKgGJEUYhtUY9GuoI/tp4mv6CzxHfXl/Zi 24 | KkpEGAA0hk8UxsBF6UbwMi7gEEcazlLik1gHfhECgYEA0HyiW3LaWfACD2y7gpYu 25 | GDvF+Oo28tK2QBkPBa1FNtZ4BKBquGqe8V6iNuQ5HsZFwycFum/VU0FWoDOyLS5M 26 | vclQT0fojPxlLTjS0B2PRBEv52cNNFwMrj/I/DerqdDh3saUIe1MeJoVIMDTE/Jl 27 | H5v0FaUgorY35MZdyQ54uw0= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /tests/data/generate-crts/ca.srl: -------------------------------------------------------------------------------- 1 | 51042C938C1D19FADAC29CA05FAE1A16D47AFF53 2 | -------------------------------------------------------------------------------- /tests/data/generate-crts/generate-test-certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CA_KEY="ca.key" 4 | CA_CERT="ca.crt" 5 | 6 | # Output paths for the database and storage certificates and keys 7 | DATABASE_KEY="../database.key" 8 | DATABASE_CSR="database.csr" 9 | DATABASE_CERT="../database.crt" 10 | 11 | STORAGE_KEY="../storage.key" 12 | STORAGE_CSR="storage.csr" 13 | STORAGE_CERT="../storage.crt" 14 | 15 | generate_certificate() { 16 | local KEY_PATH=$1 17 | local CSR_PATH=$2 18 | local CERT_PATH=$3 19 | local CONFIG_FILE=$4 20 | 21 | openssl req -new -newkey rsa:2048 -nodes -keyout "$KEY_PATH" -out "$CSR_PATH" -config "$CONFIG_FILE" 22 | openssl x509 -req -in "$CSR_PATH" -CA "$CA_CERT" -CAkey "$CA_KEY" -CAcreateserial -out "$CERT_PATH" -days 5475 -sha256 -extensions req_ext -extfile "$CONFIG_FILE" 23 | } 24 | 25 | # Paths to .cnf files, where we will write certificate settings 26 | DATABASE_CONFIG="database-csr.cnf" 27 | STORAGE_CONFIG="storage-csr.cnf" 28 | 29 | cat > $DATABASE_CONFIG < $STORAGE_CONFIG <