├── .actionlint.yaml ├── .dockerignore ├── .envrc.sample ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── 01-normal-issue.md │ ├── 02-bug_report.yml │ ├── bug_report.yml │ ├── config.yml │ ├── new_version.md │ └── normal-issue.md ├── PULL_REQUEST_TEMPLATE │ ├── pre-release-getting-started-script.md │ └── pre-release-rust-deps.md ├── actionlint.yaml ├── pull_request_template.md └── workflows │ ├── build.yml │ ├── general_daily_security.yml │ ├── integration-test.yml │ └── pr_pre-commit.yaml ├── .gitignore ├── .hadolint.yaml ├── .markdownlint.yaml ├── .pre-commit-config.yaml ├── .pylintrc ├── .readme ├── README.md.j2 ├── partials │ ├── borrowed │ │ ├── documentation.md.j2 │ │ ├── footer.md.j2 │ │ ├── header.md.j2 │ │ ├── links.md.j2 │ │ ├── overview_blurb.md.j2 │ │ └── related_reading.md.j2 │ └── main.md.j2 └── static │ └── borrowed │ ├── Icon_Stackable.svg │ ├── sdp_overview.png │ └── stackable_overview.png ├── .vscode ├── launch.json └── settings.json ├── .yamllint.yaml ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.nix ├── Cargo.toml ├── Justfile ├── LICENSE ├── Makefile ├── README.md ├── Tiltfile ├── crate-hashes.json ├── default.nix ├── deny.toml ├── deploy ├── DO_NOT_EDIT.md ├── config-spec │ └── properties.yaml ├── helm │ ├── chart_testing.yaml │ ├── ct.yaml │ └── druid-operator │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── README.md │ │ ├── configs │ │ └── properties.yaml │ │ ├── crds │ │ └── crds.yaml │ │ ├── templates │ │ ├── _helpers.tpl │ │ ├── _telemetry.tpl │ │ ├── configmap.yaml │ │ ├── deployment.yaml │ │ ├── roles.yaml │ │ └── serviceaccount.yaml │ │ └── values.yaml └── stackable-operators-ns.yaml ├── dev-cluster.yml ├── docker └── Dockerfile ├── docs ├── antora.yml ├── modules │ └── druid │ │ ├── examples │ │ ├── druid-ldap-authentication.yaml │ │ ├── druid-oidc-authentication.yaml │ │ └── getting_started │ │ │ ├── druid.yaml │ │ │ ├── druid.yaml.j2 │ │ │ ├── expected_query_result.json │ │ │ ├── getting_started.sh │ │ │ ├── getting_started.sh.j2 │ │ │ ├── hdfs.yaml │ │ │ ├── hdfs.yaml.j2 │ │ │ ├── ingestion_spec.json │ │ │ ├── install_output.txt │ │ │ ├── install_output.txt.j2 │ │ │ ├── query.json │ │ │ ├── test_getting_started_helm.sh │ │ │ ├── test_getting_started_stackablectl.sh │ │ │ ├── zookeeper.yaml │ │ │ └── zookeeper.yaml.j2 │ │ ├── images │ │ ├── druid_overview.drawio.svg │ │ └── getting_started │ │ │ ├── dashboard.png │ │ │ ├── load_example.png │ │ │ ├── load_success.png │ │ │ └── query.png │ │ ├── pages │ │ ├── getting_started │ │ │ ├── first_steps.adoc │ │ │ ├── index.adoc │ │ │ └── installation.adoc │ │ ├── index.adoc │ │ ├── reference │ │ │ ├── commandline-parameters.adoc │ │ │ ├── crds.adoc │ │ │ ├── discovery.adoc │ │ │ ├── environment-variables.adoc │ │ │ └── index.adoc │ │ ├── required-external-components.adoc │ │ └── usage-guide │ │ │ ├── deep-storage.adoc │ │ │ ├── extensions.adoc │ │ │ ├── index.adoc │ │ │ ├── ingestion.adoc │ │ │ ├── listenerclass.adoc │ │ │ ├── logging.adoc │ │ │ ├── monitoring.adoc │ │ │ ├── operations │ │ │ ├── cluster-operations.adoc │ │ │ ├── graceful-shutdown.adoc │ │ │ ├── index.adoc │ │ │ ├── pod-disruptions.adoc │ │ │ └── pod-placement.adoc │ │ │ ├── overrides.adoc │ │ │ ├── resources-and-storage.adoc │ │ │ └── security.adoc │ │ └── partials │ │ ├── nav.adoc │ │ ├── s3-credentials.adoc │ │ ├── s3-note.adoc │ │ └── supported-versions.adoc └── templating_vars.yaml ├── examples ├── psql-s3 │ ├── README.md │ └── psql-s3-druid-cluster.yaml ├── psql │ ├── README.md │ └── psql-hdfs-druid-cluster.yaml └── tls │ └── tls-druid-cluster.yaml ├── nix ├── README.md ├── meta.json ├── sources.json └── sources.nix ├── renovate.json ├── rust-toolchain.toml ├── rust └── operator-binary │ ├── Cargo.toml │ ├── build.rs │ ├── src │ ├── authentication │ │ ├── ldap.rs │ │ ├── mod.rs │ │ └── oidc.rs │ ├── config │ │ ├── jvm.rs │ │ └── mod.rs │ ├── crd │ │ ├── affinity.rs │ │ ├── authentication.rs │ │ ├── authorization.rs │ │ ├── memory.rs │ │ ├── mod.rs │ │ ├── resource.rs │ │ ├── security.rs │ │ ├── storage.rs │ │ └── tls.rs │ ├── discovery.rs │ ├── druid_controller.rs │ ├── extensions.rs │ ├── internal_secret.rs │ ├── main.rs │ ├── operations │ │ ├── graceful_shutdown.rs │ │ ├── mod.rs │ │ └── pdb.rs │ └── product_logging.rs │ └── test │ └── resources │ ├── crd │ ├── resource_merge │ │ ├── druid_cluster.yaml │ │ └── segment_cache.yaml │ └── role_service │ │ └── druid_cluster.yaml │ └── druid_controller │ ├── properties.yaml │ ├── segment_cache.yaml │ └── simple.yaml ├── rustfmt.toml ├── scripts ├── docs_templating.sh ├── ensure_one_trailing_newline.py ├── generate-manifests.sh ├── render_readme.sh ├── run-tests └── run_tests.sh ├── shell.nix └── tests ├── README-templating.md ├── infrastructure.yaml ├── kuttl-test.yaml.jinja2 ├── release.yaml ├── templates ├── .gitkeep └── kuttl │ ├── authorizer │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 01-assert.yaml │ ├── 01-install-zk.yaml.j2 │ ├── 02-assert.yaml │ ├── 02-install-opa.yaml.j2 │ ├── 03-assert.yaml │ ├── 03-install-hdfs.yaml.j2 │ ├── 04-assert.yaml │ ├── 04-install-druid.yaml.j2 │ ├── 05-assert.yaml │ ├── 05-checks-container.yaml │ ├── 06-assert.yaml │ ├── 06-authcheck.yaml │ ├── README.md │ └── authcheck.py │ ├── cluster-operation │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 10-assert.yaml │ ├── 10-install-zk.yaml.j2 │ ├── 20-assert.yaml │ ├── 20-install-hdfs.yaml.j2 │ ├── 30-assert.yaml │ ├── 30-install-druid.yaml.j2 │ ├── 40-assert.yaml │ ├── 40-stop-druid.yaml.j2 │ ├── 50-assert.yaml │ ├── 50-pause-druid.yaml.j2 │ ├── 60-assert.yaml │ └── 60-restart-druid.yaml.j2 │ ├── commons │ ├── druid-quickstartimport.json │ ├── healthcheck.py │ ├── ingestioncheck-tls.py │ └── ingestioncheck.py │ ├── hdfs-deep-storage │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 01-assert.yaml │ ├── 01-install-zk.yaml.j2 │ ├── 02-assert.yaml │ ├── 02-install-hdfs.yaml.j2 │ ├── 03-assert.yaml │ ├── 03-install-druid.yaml.j2 │ ├── 04-assert.yaml │ ├── 04-checks-container.yaml │ ├── 05-assert.yaml │ ├── 05-healthcheck.yaml │ ├── 06-assert.yaml │ ├── 06-ingestioncheck.yaml │ └── 07-assert.yaml │ ├── ingestion-no-s3-ext │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 01-assert.yaml │ ├── 01-install-zk.yaml.j2 │ ├── 02-assert.yaml │ ├── 02-install-hdfs.yaml.j2 │ ├── 03-assert.yaml │ ├── 03-install-druid.yaml.j2 │ ├── 04-assert.yaml │ ├── 04-checks-container.yaml │ ├── 05-assert.yaml │ ├── 05-healthcheck.yaml │ ├── 06-assert.yaml │ └── 06-ingestioncheck.yaml │ ├── ingestion-s3-ext │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 01-assert.yaml │ ├── 01-install-zk.yaml.j2 │ ├── 02-assert.yaml │ ├── 02-install-hdfs.yaml.j2 │ ├── 03-assert.yaml │ ├── 03-install-druid.yaml.j2 │ ├── 04-assert.yaml │ ├── 04-checks-container.yaml │ ├── 05-assert.yaml │ ├── 05-healthcheck.yaml │ ├── 06-assert.yaml │ └── 06-ingestioncheck.yaml │ ├── ldap │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 01-assert.yaml │ ├── 01-install-openldap.yaml │ ├── 01_openldap.yaml.j2 │ ├── 02-assert.yaml │ ├── 02-create-ldap-user.yaml │ ├── 03-assert.yaml │ ├── 03-install-test-druid.yaml │ ├── 10-assert.yaml │ ├── 10-install-zk.yaml.j2 │ ├── 11-create-authentication-classes.yaml.j2 │ ├── 11_authentication-classes.yaml.j2 │ ├── 12-assert.yaml │ ├── 12-install-hdfs.yaml.j2 │ ├── 13-assert.yaml │ ├── 13-install-druid.yaml.j2 │ ├── 20-assert.yaml │ ├── 20-authcheck.yaml │ ├── authcheck.py │ └── create_ldap_user.sh │ ├── logging │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 01-assert.yaml │ ├── 01-install-druid-vector-aggregator.yaml │ ├── 01_druid-vector-aggregator-values.yaml.j2 │ ├── 02-assert.yaml │ ├── 02-install-zk.yaml.j2 │ ├── 03-assert.yaml │ ├── 03-install-hdfs.yaml.j2 │ ├── 04-create-configmap-with-prepared-logs.yaml │ ├── 05-assert.yaml │ ├── 05-install-druid.yaml.j2 │ ├── 06-assert.yaml │ ├── 06-install-druid-test-runner.yaml │ ├── 07-assert.yaml │ ├── 07-test-log-aggregation.yaml │ ├── prepared-logs.log4j2.xml │ └── test_log_aggregation.py │ ├── oidc │ ├── 00-druid-secrets.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 00-s3-secret.yaml.j2 │ ├── 04-install-minio.yaml.j2 │ ├── 04_helm-bitnami-minio-values.yaml.j2 │ ├── 10-assert.yaml.j2 │ ├── 10-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 20-assert.yaml │ ├── 20-zookeeper.yaml.j2 │ ├── 30-assert.yaml │ ├── 30-install-keycloak.yaml │ ├── 30_keycloak.yaml.j2 │ ├── 35-install-authentication-class.yaml.j2 │ ├── 35_authentication-class.yaml.j2 │ ├── 40-assert.yaml │ ├── 40-install-druid.yaml │ ├── 40_druid.yaml.j2 │ ├── 50-create-configmap.yaml.j2 │ ├── 51-assert.yaml │ ├── 51-install-test-container.yaml.j2 │ ├── 60-assert.yaml │ ├── 60-login.yaml │ ├── 70-assert.yaml.j2 │ └── login.py │ ├── orphaned-resources │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 01-assert.yaml │ ├── 01-install-zk.yaml.j2 │ ├── 02-assert.yaml │ ├── 02-install-hdfs.yaml.j2 │ ├── 03-assert.yaml │ ├── 03-install-druid.yaml.j2 │ ├── 04-assert.yaml │ ├── 04-errors.yaml │ └── 04-update-druid.yaml.j2 │ ├── overrides │ ├── 00-assert.yaml.j2 │ ├── 00-install-zk.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 10-assert.yaml │ ├── 10-install-hdfs.yaml.j2 │ ├── 20-assert.yaml │ ├── 20-install-druid.yaml.j2 │ ├── 21-assert.yaml │ └── 22-assert.yaml │ ├── resources │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 00-s3-secret.yaml.j2 │ ├── 02-install-minio.yaml.j2 │ ├── 02_helm-bitnami-minio-values.yaml.j2 │ ├── 10-assert.yaml │ ├── 10-install-zk.yaml.j2 │ ├── 30-assert.yaml.j2 │ ├── 30-install-druid.yaml.j2 │ └── 40-assert.yaml │ ├── s3-deep-storage │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 00-s3-secret.yaml.j2 │ ├── 01-assert.yaml │ ├── 01-install-zk.yaml.j2 │ ├── 04-install-minio.yaml.j2 │ ├── 04_helm-bitnami-minio-values.yaml.j2 │ ├── 07-assert.yaml │ ├── 07-checks-container.yaml │ ├── 10-assert.yaml │ ├── 10-install-druid.yaml.j2 │ ├── 11-assert.yaml │ ├── 11-healthcheck.yaml │ ├── 12-assert.yaml │ ├── 12-ingestioncheck.yaml │ └── certs │ │ ├── ca.crt │ │ ├── client.crt.pem │ │ ├── client.csr.pem │ │ ├── client.key.pem │ │ ├── generate.sh │ │ ├── root-ca.crt.pem │ │ ├── root-ca.crt.srl │ │ ├── root-ca.key.pem │ │ ├── tls.crt │ │ └── tls.key │ ├── smoke │ ├── 00-limit-range.yaml │ ├── 00-patch-ns.yaml.j2 │ ├── 10-assert.yaml.j2 │ ├── 10-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 20-assert.yaml │ ├── 20-install-postgres.yaml │ ├── 20_helm-bitnami-postgresql-values.yaml.j2 │ ├── 30-assert.yaml │ ├── 30-install-zk.yaml.j2 │ ├── 40-assert.yaml │ ├── 40-install-hdfs.yaml.j2 │ ├── 50-assert.yaml │ ├── 50-install-druid.yaml.j2 │ ├── 51-assert.yaml │ ├── 60-assert.yaml │ ├── 60-checks-container.yaml │ ├── 70-assert.yaml │ └── 70-healthcheck.yaml │ └── tls │ ├── 00-assert.yaml.j2 │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 00-patch-ns.yaml.j2 │ ├── 00-s3-secret.yaml.j2 │ ├── 01-assert.yaml │ ├── 01-install-zk.yaml.j2 │ ├── 03-install-minio.yaml.j2 │ ├── 03_helm-bitnami-minio-values.yaml.j2 │ ├── 04-assert.yaml.j2 │ ├── 04-install-druid.yaml.j2 │ ├── 05-assert.yaml │ ├── 05-install-checks.yaml.j2 │ ├── 10-assert.yaml.j2 │ ├── 10-tls-checks.yaml │ ├── 11-assert.yaml.j2 │ ├── 11-ingestion-checks.yaml │ ├── check-tls.sh │ └── untrusted-ca.crt └── test-definition.yaml /.actionlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | self-hosted-runner: 3 | # Ubicloud machines we are using 4 | labels: 5 | - ubicloud-standard-8-arm 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | debug/ 2 | target/ 3 | **/*.rs.bk 4 | 5 | .idea/ 6 | *.iws 7 | 8 | Cargo.nix 9 | crate-hashes.json 10 | result 11 | image.tar 12 | 13 | # We do NOT want to ignore .git because we use the `built` crate to gather the current git commit hash at built time 14 | # This means we need the .git directory in our Docker image, it will be thrown away and won't be included in the final image 15 | -------------------------------------------------------------------------------- /.envrc.sample: -------------------------------------------------------------------------------- 1 | # vim: syntax=conf 2 | # 3 | # If you use direnv, you can autoload the nix shell: 4 | # You will need to allow the directory the first time. 5 | use nix 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | nix/** linguist-generated 2 | Cargo.nix linguist-generated 3 | crate-hashes.json linguist-generated 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/01-normal-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Normal issue 3 | about: This is just a normal empty issue with a simple checklist 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Issue checklist 11 | 12 | This is a simple checklist of things to bear in mind when creating a new issue. 13 | 14 | - [ ] **Describe the use-case**: As far as possible, use the pattern "As a [type of user], I would like [feature/functionality] to be able to do [specific action]." This helps identify the feature and the problem it addresses. 15 | - [ ] **Indicate importance and urgency**: Use a scale (e.g., low, medium, high) to indicate the level of importance and urgency. 16 | - [ ] **Work-around**: If there is a known work-around, describe it briefly. 17 | - [ ] **Environment**: Describe the environment where the issue occurs (e.g., SDP version, K8S version, etc.). 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | blank_issues_enabled: true 3 | contact_links: 4 | - name: 🙋🏾 Question 5 | about: Use this to ask a question about this project 6 | url: https://github.com/orgs/stackabletech/discussions/new?category=q-a 7 | - name: 🚀 Feature Requests and other things 8 | about: Open an issue with your feature request or any other issue not covered elsewhere 9 | url: https://github.com/stackabletech/druid-operator/issues/new 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/new_version.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: New Version 3 | about: Request support for a new product version 4 | title: "[NEW VERSION]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Which new version of Apache Druid should we support? 11 | 12 | Please specify the version, version range or version numbers to support, please also add these to the issue title 13 | 14 | ## Additional information 15 | 16 | If possible, provide a link to release notes/changelog 17 | 18 | ## Changes required 19 | 20 | Are there any upstream changes that we need to support? 21 | e.g. new features, changed features, deprecated features etc. 22 | 23 | ## Implementation checklist 24 | 25 | 29 | 30 | - [ ] Update the Docker image 31 | - [ ] Update documentation to include supported version(s) 32 | - [ ] Update and test getting started guide with updated version(s) 33 | - [ ] Update operator to support the new version (if needed) 34 | - [ ] Update integration tests to test use the new versions (in addition or replacing old versions 35 | - [ ] Update examples to use new versions 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/normal-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Normal issue 3 | about: This is just a normal empty issue with a simple checklist 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Issue checklist 11 | 12 | This is a simple checklist of things to bear in mind when creating a new issue. 13 | 14 | - [ ] Describe the use-case, as far is possible. For instance, using the pattern "As a XXXX, I would like XXXX to be able to do XXXX" helps to identify the feature as well as the problem it is intended to address. 15 | - [ ] Indicate an approximate level of importance and urgency. 16 | - [ ] Indicate if there is a known work-around until such time as the issue has been implemented. 17 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pre-release-getting-started-script.md: -------------------------------------------------------------------------------- 1 | ## Check and Update Getting Started Script 2 | 3 | 7 | 8 | 11 | 12 | Part of 13 | 14 | > [!NOTE] 15 | > During a Stackable release we need to check (and optionally update) the 16 | > getting-started scripts to ensure they still work after product and operator 17 | > updates. 18 | 19 | ```shell 20 | # Some of the scripts are in a code/ subdirectory 21 | # pushd docs/modules/superset/examples/getting_started 22 | # pushd docs/modules/superset/examples/getting_started/code 23 | pushd $(fd -td getting_started | grep examples); cd code 2>/dev/null || true 24 | 25 | # Make a fresh cluster (~12 seconds) 26 | kind delete cluster && kind create cluster 27 | ./getting_started.sh stackablectl 28 | 29 | # Make a fresh cluster (~12 seconds) 30 | kind delete cluster && kind create cluster 31 | ./getting_started.sh helm 32 | 33 | popd 34 | ``` 35 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pre-release-rust-deps.md: -------------------------------------------------------------------------------- 1 | ## Bump Rust Dependencies for Stackable Release YY.M.X 2 | 3 | 7 | 8 | 11 | 12 | Part of 13 | 14 | > [!NOTE] 15 | > During a Stackable release we need to update various Rust dependencies before 16 | > entering the final release period to ensure we run the latest versions of 17 | > crates. These bumps also include previously updated and released crates from 18 | > the `operator-rs` repository. 19 | 20 | ### Tasks 21 | 22 | - [ ] Bump Rust Dependencies, see below for more details. 23 | - [ ] Add changelog entry stating which important crates were bumped (including the version). 24 | 25 | > [!NOTE] 26 | > The bumping / updating of Rust dependencies is done in multiple steps: 27 | > 28 | > 1. Update the minimum Version in the root `Cargo.toml` manifest. 29 | > 2. Run the `cargo update` command, which also updates the `Cargo.lock` file. 30 | > 3. Lastly, run `make regenerate-nix` to update the `Cargo.nix` file. 31 | 32 | ### Bump Rust Dependencies 33 | 34 | - [ ] Bump `stackable-operator` and friends 35 | - [ ] Bump `product-config` 36 | - [ ] Bump all other dependencies 37 | -------------------------------------------------------------------------------- /.github/actionlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | self-hosted-runner: 3 | # Ubicloud machines we are using 4 | labels: 5 | - ubicloud-standard-8-arm 6 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | *Please add a description here. This will become the commit message of the merge request later.* 4 | 5 | ## Definition of Done Checklist 6 | 7 | - Not all of these items are applicable to all PRs, the author should update this template to only leave the boxes in that are relevant 8 | - Please make sure all these things are done and tick the boxes 9 | 10 | ### Author 11 | 12 | - [ ] Changes are OpenShift compatible 13 | - [ ] CRD changes approved 14 | - [ ] CRD documentation for all fields, following the [style guide](https://docs.stackable.tech/home/nightly/contributor/docs/style-guide). 15 | - [ ] Helm chart can be installed and deployed operator works 16 | - [ ] Integration tests passed (for non trivial changes) 17 | - [ ] Changes need to be "offline" compatible 18 | - [ ] Links to generated (nightly) docs added 19 | - [ ] Release note snippet added 20 | 21 | ### Reviewer 22 | 23 | - [ ] Code contains useful comments 24 | - [ ] Code contains useful logging statements 25 | - [ ] (Integration-)Test cases added 26 | - [ ] Documentation added or updated. Follows the [style guide](https://docs.stackable.tech/home/nightly/contributor/docs/style-guide). 27 | - [ ] Changelog updated 28 | - [ ] Cargo.toml only contains references to git tags (not specific commits or branches) 29 | 30 | ### Acceptance 31 | 32 | - [ ] Feature Tracker has been updated 33 | - [ ] Proper release label has been added 34 | - [ ] Links to generated (nightly) docs added 35 | - [ ] Release note snippet added 36 | - [ ] Add `type/deprecation` label & add to the [deprecation schedule](https://github.com/orgs/stackabletech/projects/44/views/1) 37 | - [ ] Add `type/experimental` label & add to the [experimental features tracker](https://github.com/orgs/stackabletech/projects/47) 38 | -------------------------------------------------------------------------------- /.github/workflows/general_daily_security.yml: -------------------------------------------------------------------------------- 1 | # ============= 2 | # This file is automatically generated from the templates in stackabletech/operator-templating 3 | # DON'T MANUALLY EDIT THIS FILE 4 | # ============= 5 | --- 6 | name: Daily Security Audit 7 | 8 | on: 9 | schedule: 10 | - cron: '15 4 * * *' 11 | workflow_dispatch: 12 | 13 | permissions: {} 14 | 15 | jobs: 16 | audit: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 20 | with: 21 | persist-credentials: false 22 | - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 23 | with: 24 | token: ${{ secrets.GITHUB_TOKEN }} 25 | -------------------------------------------------------------------------------- /.github/workflows/pr_pre-commit.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: pre-commit 3 | 4 | on: 5 | pull_request: 6 | merge_group: 7 | 8 | env: 9 | CARGO_TERM_COLOR: always 10 | NIX_PKG_MANAGER_VERSION: "2.28.3" 11 | RUST_TOOLCHAIN_VERSION: "nightly-2025-05-26" 12 | HADOLINT_VERSION: "v2.12.0" 13 | PYTHON_VERSION: "3.12" 14 | 15 | jobs: 16 | pre-commit: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Install host dependencies 20 | uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3 21 | with: 22 | packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https 23 | version: ubuntu-latest 24 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 25 | with: 26 | persist-credentials: false 27 | submodules: recursive 28 | fetch-depth: 0 29 | - uses: stackabletech/actions/run-pre-commit@9aae2d1c14239021bfa33c041010f6fb7adec815 # v0.8.2 30 | with: 31 | python-version: ${{ env.PYTHON_VERSION }} 32 | rust: ${{ env.RUST_TOOLCHAIN_VERSION }} 33 | hadolint: ${{ env.HADOLINT_VERSION }} 34 | nix: ${{ env.NIX_PKG_MANAGER_VERSION }} 35 | nix-github-token: ${{ secrets.GITHUB_TOKEN }} 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tests/_work/ 2 | debug/ 3 | target/ 4 | **/*.rs.bk 5 | 6 | .idea/ 7 | *.iws 8 | *.iml 9 | 10 | *.tgz 11 | 12 | result 13 | image.tar 14 | 15 | tilt_options.json 16 | 17 | .direnv/ 18 | .direnvrc 19 | .envrc 20 | 21 | .DS_Store 22 | -------------------------------------------------------------------------------- /.hadolint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ignored: 3 | # Warning: Use the -y switch to avoid manual input dnf install -y 4 | # https://github.com/hadolint/hadolint/wiki/DL3038 5 | # Reason: We set `assumeyes=True` in dnf.conf in our base image 6 | - DL3038 7 | 8 | # Warning: Specify version with dnf install -y - 9 | # https://github.com/hadolint/hadolint/wiki/DL3041 10 | # Reason: It's good advice, but we're not set up to pin versions just yet 11 | - DL3041 12 | -------------------------------------------------------------------------------- /.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # All defaults or options can be checked here: 3 | # https://github.com/DavidAnson/markdownlint/blob/main/schema/.markdownlint.yaml 4 | 5 | # Default state for all rules 6 | default: true 7 | 8 | # MD013/line-length - Line length 9 | MD013: 10 | # Number of characters 11 | line_length: 9999 12 | # Number of characters for headings 13 | heading_line_length: 9999 14 | # Number of characters for code blocks 15 | code_block_line_length: 9999 16 | 17 | # MD033/no-inline-html 18 | MD033: 19 | allowed_elements: [h1, img, p] 20 | 21 | # MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the same content 22 | MD024: 23 | # Only check sibling headings 24 | siblings_only: true 25 | 26 | # MD041/first-line-heading/first-line-h1 First line in a file should be a top-level heading 27 | MD041: false # Github issues and PRs already have titles, and H1 is enormous in the description box. 28 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | 3 | # These rules are for missing docstrings which doesn't matter much for most of our simple scripts 4 | disable=C0114,C0115,C0116 5 | 6 | [FORMAT] 7 | 8 | max-line-length=999 9 | indent-string=' ' 10 | -------------------------------------------------------------------------------- /.readme/README.md.j2: -------------------------------------------------------------------------------- 1 | {%- set title="Stackable Operator for Apache Druid" -%} 2 | {%- set operator_name="druid" -%} 3 | {%- set operator_docs_slug="druid" -%} 4 | {%- set related_reading_links=[] -%} 5 | 6 | {% filter trim %} 7 | {%- include "partials/borrowed/header.md.j2" -%} 8 | {% endfilter %} 9 | 10 | {% filter trim %} 11 | {%- include "partials/borrowed/links.md.j2" -%} 12 | {% endfilter %} 13 | 14 | {% filter trim %} 15 | {%- include "partials/main.md.j2" -%} 16 | {% endfilter %} 17 | 18 | {% filter trim %} 19 | {%- include "partials/borrowed/footer.md.j2" -%} 20 | {% endfilter %} 21 | 22 | {% filter trim %} 23 | {%- include "partials/borrowed/related_reading.md.j2" -%} 24 | {% endfilter %} 25 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/documentation.md.j2: -------------------------------------------------------------------------------- 1 | 2 | ## Documentation 3 | 4 | The stable documentation for this operator can be found in our [Stackable Data Platform documentation](https://docs.stackable.tech/home/stable/{{operator_docs_slug}}). 5 | If you are interested in the most recent state of this repository, check out the [nightly docs](https://docs.stackable.tech/home/nightly/{{operator_docs_slug}}) instead. 6 | 7 | The documentation for all Stackable products can be found at [docs.stackable.tech](https://docs.stackable.tech). 8 | 9 | If you have a question about the Stackable Data Platform, contact us via our [homepage](https://stackable.tech/) or ask a public question in our [Discussions forum](https://github.com/orgs/stackabletech/discussions). 10 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/header.md.j2: -------------------------------------------------------------------------------- 1 | 2 |

3 | Stackable Logo 4 |

5 | 6 |

{{title}}

7 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/links.md.j2: -------------------------------------------------------------------------------- 1 | 2 | [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/stackabletech/{{operator_name}}-operator/graphs/commit-activity) 3 | [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-green.svg)](https://docs.stackable.tech/home/stable/contributor/index.html) 4 | [![License OSL3.0](https://img.shields.io/badge/license-OSL3.0-green)](./LICENSE) 5 | 6 | [Documentation](https://docs.stackable.tech/home/stable/{{operator_docs_slug}}) {% if quickstart_link %}| [Quickstart]({{quickstart_link}}) {% endif %}| [Stackable Data Platform](https://stackable.tech/) | [Platform Docs](https://docs.stackable.tech/) | [Discussions](https://github.com/orgs/stackabletech/discussions) | [Discord](https://discord.gg/7kZ3BNnCAF) 7 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/overview_blurb.md.j2: -------------------------------------------------------------------------------- 1 | 2 | It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka, Apache Druid, Trino or Apache Spark, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms). 3 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/related_reading.md.j2: -------------------------------------------------------------------------------- 1 | 2 | {%- if related_reading_links -%} 3 | ## Related Reading 4 | {% for (text, link) in related_reading_links %} 5 | * [{{text}}]({{link}}) 6 | {%- endfor %} 7 | {%- endif -%} 8 | -------------------------------------------------------------------------------- /.readme/partials/main.md.j2: -------------------------------------------------------------------------------- 1 | This is a Kubernetes operator to manage [Apache Druid](https://druid.apache.org/) ensembles. 2 | 3 | {% filter trim %} 4 | {%- include "partials/borrowed/overview_blurb.md.j2" -%} 5 | {% endfilter %} 6 | 7 | ## Installation 8 | 9 | You can install the operator using [stackablectl or helm](https://docs.stackable.tech/home/stable/{{operator_name}}/getting_started/installation). 10 | 11 | Read on to get started with it, or see it in action in one of our [demos](https://stackable.tech/en/demos/). 12 | 13 | ## Getting Started 14 | 15 | You can follow this [tutorial](https://docs.stackable.tech/home/stable/{{operator_name}}/getting_started/first_steps). 16 | 17 | {% filter trim %} 18 | {%- include "partials/borrowed/documentation.md.j2" -%} 19 | {% endfilter %} 20 | -------------------------------------------------------------------------------- /.readme/static/borrowed/sdp_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/druid-operator/26e58068bbf7ed13cb89b70bf474336dd7190097/.readme/static/borrowed/sdp_overview.png -------------------------------------------------------------------------------- /.readme/static/borrowed/stackable_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/druid-operator/26e58068bbf7ed13cb89b70bf474336dd7190097/.readme/static/borrowed/stackable_overview.png -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "lldb", 6 | "request": "launch", 7 | "name": "Debug operator binary", 8 | "cargo": { 9 | "args": ["build"], 10 | "filter": { 11 | "name": "stackable-{[ operator.name }]", 12 | "kind": "bin" 13 | } 14 | }, 15 | "args": ["run"], 16 | "cwd": "${workspaceFolder}" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rust-analyzer.rustfmt.overrideCommand": [ 3 | "rustfmt", 4 | "+nightly-2025-05-26", 5 | "--edition", 6 | "2024", 7 | "--" 8 | ], 9 | } 10 | -------------------------------------------------------------------------------- /.yamllint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | ignore: | 5 | deploy/helm/**/templates 6 | 7 | rules: 8 | line-length: disable 9 | truthy: 10 | check-keys: false 11 | comments: 12 | min-spaces-from-content: 1 # Needed due to https://github.com/adrienverge/yamllint/issues/443 13 | indentation: 14 | indent-sequences: consistent 15 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["rust/operator-binary"] 3 | resolver = "2" 4 | 5 | [workspace.package] 6 | version = "0.0.0-dev" 7 | authors = ["Stackable GmbH "] 8 | license = "OSL-3.0" 9 | edition = "2021" 10 | repository = "https://github.com/stackabletech/druid-operator" 11 | 12 | [workspace.dependencies] 13 | product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } 14 | stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.93.0" } 15 | 16 | anyhow = "1.0" 17 | built = { version = "0.8", features = ["chrono", "git2"] } 18 | clap = "4.5" 19 | const_format = "0.2" 20 | fnv = "1.0" 21 | futures = { version = "0.3", features = ["compat"] } 22 | indoc = "2.0" 23 | openssl = "0.10" 24 | pin-project = "1.1" 25 | rstest = "0.25" 26 | semver = "1.0" 27 | serde = { version = "1.0", features = ["derive"] } 28 | serde_json = "1.0" 29 | serde_yaml = "0.9" 30 | snafu = "0.8" 31 | strum = { version = "0.27", features = ["derive"] } 32 | tokio = { version = "1.40", features = ["full"] } 33 | tracing = "0.1" 34 | 35 | [patch."https://github.com/stackabletech/operator-rs.git"] 36 | # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } 37 | # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } 38 | -------------------------------------------------------------------------------- /Justfile: -------------------------------------------------------------------------------- 1 | default: 2 | @just --list 3 | 4 | actionlint: 5 | actionlint 6 | 7 | build: d 8 | cargo build 9 | 10 | build-release: 11 | cargo build --release 12 | 13 | check: actionlint clippy doc fmt test udeps 14 | 15 | clean: 16 | cargo clean 17 | 18 | clippy: 19 | cargo clippy --locked -- -D warnings 20 | 21 | doc: 22 | RUSTDOCFLAGS="-D warnings" cargo +nightly doc --document-private-items 23 | 24 | fmt: 25 | cargo fmt --all 26 | 27 | pr: 28 | gh pr create --web 29 | 30 | test: 31 | cargo test --locked 32 | 33 | udeps: 34 | cargo +nightly udeps --all-targets --backend depinfo 35 | 36 | # TODO: Stuff from Makefile 37 | -------------------------------------------------------------------------------- /crate-hashes.json: -------------------------------------------------------------------------------- 1 | { 2 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.0#k8s-version@0.1.3": "1dv5vgilcpj1h88pdzzb94aj85nrm5bm0qkpplwd5b0m857b6rmp", 3 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.0#stackable-operator-derive@0.3.1": "1dv5vgilcpj1h88pdzzb94aj85nrm5bm0qkpplwd5b0m857b6rmp", 4 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.0#stackable-operator@0.93.0": "1dv5vgilcpj1h88pdzzb94aj85nrm5bm0qkpplwd5b0m857b6rmp", 5 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.0#stackable-shared@0.0.1": "1dv5vgilcpj1h88pdzzb94aj85nrm5bm0qkpplwd5b0m857b6rmp", 6 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.0#stackable-telemetry@0.6.0": "1dv5vgilcpj1h88pdzzb94aj85nrm5bm0qkpplwd5b0m857b6rmp", 7 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.0#stackable-versioned-macros@0.7.1": "1dv5vgilcpj1h88pdzzb94aj85nrm5bm0qkpplwd5b0m857b6rmp", 8 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.0#stackable-versioned@0.7.1": "1dv5vgilcpj1h88pdzzb94aj85nrm5bm0qkpplwd5b0m857b6rmp", 9 | "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" 10 | } -------------------------------------------------------------------------------- /deploy/DO_NOT_EDIT.md: -------------------------------------------------------------------------------- 1 | # DO NOT EDIT 2 | 3 | These Helm charts and manifests are automatically generated. 4 | Please do not edit anything except for files explicitly mentioned below in this 5 | directory manually. 6 | 7 | The following files are ok to edit: 8 | 9 | - helm/druid-operator/templates/roles.yaml 10 | - helm/druid-operator/values.yaml 11 | 12 | The details are in-motion but check this repository for a few details: 13 | 14 | -------------------------------------------------------------------------------- /deploy/helm/chart_testing.yaml: -------------------------------------------------------------------------------- 1 | remote: origin 2 | target-branch: main 3 | chart-dirs: 4 | - deploy/helm 5 | all: true 6 | -------------------------------------------------------------------------------- /deploy/helm/ct.yaml: -------------------------------------------------------------------------------- 1 | # This file is used for chart-testing (https://github.com/helm/chart-testing) 2 | # The name "ct.yaml" is not very self-descriptive but it is the default that chart-testing is looking for 3 | --- 4 | remote: origin 5 | target-branch: main 6 | chart-dirs: 7 | - deploy/helm 8 | all: true 9 | -------------------------------------------------------------------------------- /deploy/helm/druid-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # ============= 2 | # This file is automatically generated from the templates in stackabletech/operator-templating 3 | # DON'T MANUALLY EDIT THIS FILE 4 | # ============= 5 | 6 | # Patterns to ignore when building packages. 7 | # This supports shell glob matching, relative path matching, and 8 | # negation (prefixed with !). Only one pattern per line. 9 | .DS_Store 10 | # Common VCS dirs 11 | .git/ 12 | .gitignore 13 | .bzr/ 14 | .bzrignore 15 | .hg/ 16 | .hgignore 17 | .svn/ 18 | # Common backup files 19 | *.swp 20 | *.bak 21 | *.tmp 22 | *.orig 23 | *~ 24 | # Various IDEs 25 | .project 26 | .idea/ 27 | *.tmproj 28 | .vscode/ 29 | -------------------------------------------------------------------------------- /deploy/helm/druid-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v2 3 | name: druid-operator 4 | version: "0.0.0-dev" 5 | appVersion: "0.0.0-dev" 6 | description: The Stackable Operator for Apache Druid 7 | home: https://github.com/stackabletech/druid-operator 8 | maintainers: 9 | - name: Stackable 10 | url: https://www.stackable.tech 11 | -------------------------------------------------------------------------------- /deploy/helm/druid-operator/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Helm Chart for Stackable Operator for Apache Druid 3 | 4 | This Helm Chart can be used to install Custom Resource Definitions and the Operator for Apache Druid provided by Stackable. 5 | 6 | ## Requirements 7 | 8 | - Create a [Kubernetes Cluster](../Readme.md) 9 | - Install [Helm](https://helm.sh/docs/intro/install/) 10 | 11 | ## Install the Stackable Operator for Apache Druid 12 | 13 | ```bash 14 | # From the root of the operator repository 15 | make compile-chart 16 | 17 | helm install druid-operator deploy/helm/druid-operator 18 | ``` 19 | 20 | ## Usage of the CRDs 21 | 22 | The usage of this operator and its CRDs is described in the [documentation](https://docs.stackable.tech/druid/index.html) 23 | 24 | The operator has example requests included in the [`/examples`](https://github.com/stackabletech/druid-operator/tree/main/examples) directory. 25 | 26 | ## Links 27 | 28 | 29 | -------------------------------------------------------------------------------- /deploy/helm/druid-operator/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | data: 4 | {{ (.Files.Glob "configs/*").AsConfig | indent 2 }} 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ include "operator.fullname" . }}-configmap 8 | labels: 9 | {{- include "operator.labels" . | nindent 4 }} 10 | -------------------------------------------------------------------------------- /deploy/helm/druid-operator/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | {{ if .Values.serviceAccount.create -}} 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: {{ include "operator.fullname" . }}-serviceaccount 7 | labels: 8 | {{- include "operator.labels" . | nindent 4 }} 9 | {{- with .Values.serviceAccount.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | # This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. 16 | kind: ClusterRoleBinding 17 | metadata: 18 | name: {{ include "operator.fullname" . }}-clusterrolebinding 19 | labels: 20 | {{- include "operator.labels" . | nindent 4 }} 21 | subjects: 22 | - kind: ServiceAccount 23 | name: {{ include "operator.fullname" . }}-serviceaccount 24 | namespace: {{ .Release.Namespace }} 25 | roleRef: 26 | kind: ClusterRole 27 | name: {{ include "operator.fullname" . }}-clusterrole 28 | apiGroup: rbac.authorization.k8s.io 29 | {{- end }} 30 | -------------------------------------------------------------------------------- /deploy/stackable-operators-ns.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: stackable-operators 6 | -------------------------------------------------------------------------------- /dev-cluster.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Create a cluster: 3 | # 4 | # kind create cluster --name=druid --config=dev-cluster.yml 5 | # 6 | # Delete cluster: 7 | # 8 | # kind delete cluster --name=druid 9 | # 10 | kind: Cluster 11 | apiVersion: kind.x-k8s.io/v1alpha4 12 | nodes: 13 | - role: control-plane 14 | - role: worker 15 | kubeadmConfigPatches: 16 | - | 17 | kind: JoinConfiguration 18 | nodeRegistration: 19 | kubeletExtraArgs: 20 | node-labels: "nodeType=druid-data" 21 | - role: worker 22 | - role: worker 23 | -------------------------------------------------------------------------------- /docs/antora.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: home 3 | version: "nightly" 4 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/druid-ldap-authentication.yaml: -------------------------------------------------------------------------------- 1 | # yamllint disable-file 2 | 3 | # tag::authclass[] 4 | apiVersion: authentication.stackable.tech/v1alpha1 5 | kind: AuthenticationClass 6 | metadata: 7 | name: ldap-auth 8 | spec: 9 | [...] 10 | #end::authclass[] 11 | 12 | # tag::druid[] 13 | apiVersion: druid.stackable.tech/v1alpha1 14 | kind: DruidCluster 15 | metadata: 16 | name: druid 17 | spec: 18 | clusterConfig: 19 | authentication: 20 | - authenticationClass: ldap-auth 21 | [...] 22 | # end::druid[] 23 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/druid-oidc-authentication.yaml: -------------------------------------------------------------------------------- 1 | # yamllint disable-file 2 | 3 | # tag::authclass[] 4 | apiVersion: authentication.stackable.tech/v1alpha1 5 | kind: AuthenticationClass 6 | metadata: 7 | name: oidc-auth 8 | spec: 9 | provider: 10 | oidc: 11 | [...] 12 | #end::authclass[] 13 | 14 | # tag::druid[] 15 | apiVersion: druid.stackable.tech/v1alpha1 16 | kind: DruidCluster 17 | metadata: 18 | name: druid 19 | spec: 20 | clusterConfig: 21 | authentication: 22 | - authenticationClass: oidc-auth 23 | oidc: 24 | clientCredentialsSecret: druid-oidc-client 25 | [...] 26 | # end::druid[] 27 | 28 | # tag::secret[] 29 | apiVersion: v1 30 | kind: Secret 31 | metadata: 32 | name: druid-oidc-client 33 | stringData: 34 | clientId: 35 | clientSecret: 36 | 37 | # end::secret[] 38 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/druid.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: druid.stackable.tech/v1alpha1 3 | kind: DruidCluster 4 | metadata: 5 | name: simple-druid 6 | spec: 7 | image: 8 | productVersion: 33.0.0 9 | clusterConfig: 10 | listenerClass: external-stable # This exposes your Stacklet outside of Kubernetes. Remove this configuration if this is not desired 11 | zookeeperConfigMapName: simple-druid-znode 12 | deepStorage: 13 | hdfs: 14 | configMapName: simple-hdfs 15 | directory: /druid 16 | metadataStorageDatabase: 17 | dbType: postgresql 18 | connString: jdbc:postgresql://postgresql-druid/druid 19 | host: postgresql-druid 20 | port: 5432 21 | credentialsSecret: druid-db-credentials 22 | brokers: 23 | roleGroups: 24 | default: 25 | replicas: 1 26 | coordinators: 27 | roleGroups: 28 | default: 29 | replicas: 1 30 | historicals: 31 | roleGroups: 32 | default: 33 | replicas: 1 34 | middleManagers: 35 | roleGroups: 36 | default: 37 | replicas: 1 38 | routers: 39 | roleGroups: 40 | default: 41 | replicas: 1 42 | --- 43 | apiVersion: v1 44 | kind: Secret 45 | metadata: 46 | name: druid-db-credentials 47 | stringData: 48 | username: druid 49 | password: druid 50 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/druid.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: druid.stackable.tech/v1alpha1 3 | kind: DruidCluster 4 | metadata: 5 | name: simple-druid 6 | spec: 7 | image: 8 | productVersion: 33.0.0 9 | clusterConfig: 10 | listenerClass: external-stable # This exposes your Stacklet outside of Kubernetes. Remove this configuration if this is not desired 11 | zookeeperConfigMapName: simple-druid-znode 12 | deepStorage: 13 | hdfs: 14 | configMapName: simple-hdfs 15 | directory: /druid 16 | metadataStorageDatabase: 17 | dbType: postgresql 18 | connString: jdbc:postgresql://postgresql-druid/druid 19 | host: postgresql-druid 20 | port: 5432 21 | credentialsSecret: druid-db-credentials 22 | brokers: 23 | roleGroups: 24 | default: 25 | replicas: 1 26 | coordinators: 27 | roleGroups: 28 | default: 29 | replicas: 1 30 | historicals: 31 | roleGroups: 32 | default: 33 | replicas: 1 34 | middleManagers: 35 | roleGroups: 36 | default: 37 | replicas: 1 38 | routers: 39 | roleGroups: 40 | default: 41 | replicas: 1 42 | --- 43 | apiVersion: v1 44 | kind: Secret 45 | metadata: 46 | name: druid-db-credentials 47 | stringData: 48 | username: druid 49 | password: druid 50 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/expected_query_result.json: -------------------------------------------------------------------------------- 1 | [{"page":"Copa América Centenario","Edits":29},{"page":"User:Cyde/List of candidates for speedy deletion/Subpage","Edits":16},{"page":"Wikipedia:Administrators' noticeboard/Incidents","Edits":16},{"page":"2016 Wimbledon Championships – Men's Singles","Edits":15},{"page":"Wikipedia:Administrator intervention against vandalism","Edits":15},{"page":"Wikipedia:Vandalismusmeldung","Edits":15},{"page":"The Winds of Winter (Game of Thrones)","Edits":12},{"page":"ولاية الجزائر","Edits":12},{"page":"Copa América","Edits":10},{"page":"Lionel Messi","Edits":10}] 2 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/hdfs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: hdfs.stackable.tech/v1alpha1 3 | kind: HdfsCluster 4 | metadata: 5 | name: simple-hdfs 6 | spec: 7 | image: 8 | productVersion: 3.4.1 9 | clusterConfig: 10 | dfsReplication: 1 11 | zookeeperConfigMapName: simple-hdfs-znode 12 | nameNodes: 13 | config: 14 | listenerClass: external-stable # This exposes your Stacklet outside of Kubernetes. Remove this configuration if this is not desired 15 | roleGroups: 16 | default: 17 | replicas: 2 18 | dataNodes: 19 | config: 20 | listenerClass: external-unstable # This exposes your Stacklet outside of Kubernetes. Remove this configuration if this is not desired 21 | roleGroups: 22 | default: 23 | replicas: 1 24 | journalNodes: 25 | roleGroups: 26 | default: 27 | replicas: 1 28 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: hdfs.stackable.tech/v1alpha1 3 | kind: HdfsCluster 4 | metadata: 5 | name: simple-hdfs 6 | spec: 7 | image: 8 | productVersion: 3.4.1 9 | clusterConfig: 10 | dfsReplication: 1 11 | zookeeperConfigMapName: simple-hdfs-znode 12 | nameNodes: 13 | config: 14 | listenerClass: external-stable # This exposes your Stacklet outside of Kubernetes. Remove this configuration if this is not desired 15 | roleGroups: 16 | default: 17 | replicas: 2 18 | dataNodes: 19 | config: 20 | listenerClass: external-unstable # This exposes your Stacklet outside of Kubernetes. Remove this configuration if this is not desired 21 | roleGroups: 22 | default: 23 | replicas: 1 24 | journalNodes: 25 | roleGroups: 26 | default: 27 | replicas: 1 28 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/install_output.txt: -------------------------------------------------------------------------------- 1 | Installed commons=0.0.0-dev operator 2 | Installed secret=0.0.0-dev operator 3 | Installed listener=0.0.0-dev operator 4 | Installed zookeeper=0.0.0-dev operator 5 | Installed hdfs=0.0.0-dev operator 6 | Installed druid=0.0.0-dev operator 7 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/install_output.txt.j2: -------------------------------------------------------------------------------- 1 | Installed commons={{ versions.commons }} operator 2 | Installed secret={{ versions.secret }} operator 3 | Installed listener={{ versions.listener }} operator 4 | Installed zookeeper={{ versions.zookeeper }} operator 5 | Installed hdfs={{ versions.hdfs }} operator 6 | Installed druid={{ versions.druid }} operator 7 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/query.json: -------------------------------------------------------------------------------- 1 | { 2 | "query": "SELECT page, COUNT(*) AS Edits FROM wikipedia GROUP BY page ORDER BY Edits DESC LIMIT 10" 3 | } 4 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/test_getting_started_helm.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "$0")" 5 | ./getting_started.sh helm 6 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/test_getting_started_stackablectl.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "$0")" 5 | ./getting_started.sh stackablectl 6 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/zookeeper.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: simple-zk 6 | spec: 7 | image: 8 | productVersion: 3.9.3 9 | servers: 10 | roleGroups: 11 | default: 12 | replicas: 1 13 | --- 14 | apiVersion: zookeeper.stackable.tech/v1alpha1 15 | kind: ZookeeperZnode 16 | metadata: 17 | name: simple-druid-znode 18 | spec: 19 | clusterRef: 20 | name: simple-zk 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: simple-hdfs-znode 26 | spec: 27 | clusterRef: 28 | name: simple-zk 29 | -------------------------------------------------------------------------------- /docs/modules/druid/examples/getting_started/zookeeper.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: simple-zk 6 | spec: 7 | image: 8 | productVersion: 3.9.3 9 | servers: 10 | roleGroups: 11 | default: 12 | replicas: 1 13 | --- 14 | apiVersion: zookeeper.stackable.tech/v1alpha1 15 | kind: ZookeeperZnode 16 | metadata: 17 | name: simple-druid-znode 18 | spec: 19 | clusterRef: 20 | name: simple-zk 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: simple-hdfs-znode 26 | spec: 27 | clusterRef: 28 | name: simple-zk 29 | -------------------------------------------------------------------------------- /docs/modules/druid/images/getting_started/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/druid-operator/26e58068bbf7ed13cb89b70bf474336dd7190097/docs/modules/druid/images/getting_started/dashboard.png -------------------------------------------------------------------------------- /docs/modules/druid/images/getting_started/load_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/druid-operator/26e58068bbf7ed13cb89b70bf474336dd7190097/docs/modules/druid/images/getting_started/load_example.png -------------------------------------------------------------------------------- /docs/modules/druid/images/getting_started/load_success.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/druid-operator/26e58068bbf7ed13cb89b70bf474336dd7190097/docs/modules/druid/images/getting_started/load_success.png -------------------------------------------------------------------------------- /docs/modules/druid/images/getting_started/query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/druid-operator/26e58068bbf7ed13cb89b70bf474336dd7190097/docs/modules/druid/images/getting_started/query.png -------------------------------------------------------------------------------- /docs/modules/druid/pages/getting_started/index.adoc: -------------------------------------------------------------------------------- 1 | = Getting started 2 | :description: Get started with Druid on Kubernetes using the Stackable Operator. Follow steps to install, configure, and query data. 3 | 4 | This guide helps you get started with Druid using the Stackable Operator 5 | It covers installing the operator and its dependencies, setting up your first Druid instance, ingesting example data, and querying that data. 6 | 7 | == Prerequisites 8 | 9 | You need: 10 | 11 | * a Kubernetes cluster 12 | * kubectl 13 | * optional: Helm 14 | 15 | Resource sizing depends on cluster type(s), usage and scope, but as a starting point the following resources are recommended as a minimum for this operator: 16 | 17 | * 0.2 cores (e.g. i5 or similar) 18 | * 256MB RAM 19 | 20 | == What's next 21 | 22 | The Guide is divided into two steps: 23 | 24 | * xref:getting_started/installation.adoc[Installing the Operators]. 25 | * xref:getting_started/first_steps.adoc[Setting up the Druid instance and loading and querying example data]. 26 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/reference/commandline-parameters.adoc: -------------------------------------------------------------------------------- 1 | = Command Line parameters 2 | 3 | This operator accepts the following command line parameters: 4 | 5 | == product-config 6 | 7 | *Default value*: `/etc/stackable/druid-operator/config-spec/properties.yaml` 8 | 9 | *Required*: false 10 | 11 | *Multiple values:* false 12 | 13 | [source] 14 | ---- 15 | stackable-druid-operator run --product-config /foo/bar/properties.yaml 16 | ---- 17 | 18 | == watch-namespace 19 | 20 | *Default value*: All namespaces 21 | 22 | *Required*: false 23 | 24 | *Multiple values:* false 25 | 26 | The operator **only** watches for resources in the provided namespace `test`: 27 | 28 | [source] 29 | ---- 30 | stackable-druid-operator run --watch-namespace test 31 | ---- 32 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/reference/crds.adoc: -------------------------------------------------------------------------------- 1 | = CRD Reference 2 | 3 | Find all CRD reference for the Stackable operator for Apache Druid at: {crd-docs-base-url}/druid-operator/{crd-docs-version}. 4 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/reference/discovery.adoc: -------------------------------------------------------------------------------- 1 | = Discovery ConfigMap 2 | :description: Reference documentation for the discovery ConfigMap generated by the Stackable Operator for Apache Druid. 3 | :keywords: reference, discovery, ConfigMap, connection 4 | :page-aliases: discovery.adoc 5 | :clusterName: simple-derby-druid 6 | :namespace: stackable 7 | :routerPort: 8888 8 | 9 | The Stackable operator for Druid publishes a xref:concepts:service_discovery.adoc[] with the following properties, where `{clusterName}` represents the name and `{namespace}` the namespace of the cluster: 10 | 11 | `DRUID_AVATICA_JDBC`:: 12 | ==== 13 | Contains the connection string for the https://calcite.apache.org/avatica/downloads/[Avatica] JDBC driver to connect to the router: 14 | [subs="normal"] 15 | jdbc:avatica:remote:url=http://{clusterName}-router.{namespace}.svc.cluster.local:{routerPort}/druid/v2/sql/avatica/ 16 | ==== 17 | 18 | `DRUID_SQLALCHEMY`:: 19 | ==== 20 | Contains the connection string for the python SQLAlchemy toolkit: 21 | 22 | [subs="normal"] 23 | druid://{clusterName}-router.{namespace}.svc.cluster.local:{routerPort}/druid/v2/sql 24 | ==== 25 | 26 | `DRUID_ROUTER`:: 27 | ==== 28 | Contains the connection string for the Druid router: 29 | [subs="normal"] 30 | {clusterName}-router.{namespace}.svc.cluster.local:{routerPort} 31 | ==== 32 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/reference/index.adoc: -------------------------------------------------------------------------------- 1 | = Reference 2 | 3 | Consult the reference documentation section to find exhaustive information on: 4 | 5 | * Descriptions and default values of all properties in the CRDs used by this operator in the xref:reference/crds.adoc[]. 6 | * The properties in the xref:reference/discovery.adoc[]. 7 | * The xref:reference/commandline-parameters.adoc[] and xref:reference/environment-variables.adoc[] accepted by the operator. 8 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/required-external-components.adoc: -------------------------------------------------------------------------------- 1 | = Required external components 2 | :description: Druid requires an SQL database for metadata and supports various deep storage options like S3, HDFS, and cloud storage 3 | :druid-available-metadata-stores: https://druid.apache.org/docs/latest/design/metadata-storage/#available-metadata-stores 4 | :druid-deep-storage: https://druid.apache.org/docs/latest/design/deep-storage 5 | 6 | Druid uses an SQL database to store metadata. 7 | Consult the {druid-available-metadata-stores}[Druid documentation] for a list of supported databases and setup instructions. 8 | 9 | == Feature specific: S3 and cloud deep storage 10 | 11 | {druid-deep-storage}[Deep storage] is where segments are stored. 12 | Druid offers multiple storage backends, of which we support HDFS and S3. 13 | HDFS deep storage can be set up with the xref:hdfs:index.adoc[Stackable Operator for Apache HDFS]. 14 | For S3 deep storage or the Google Cloud and Azure storage backends, you need to set up the respective storage backend. 15 | 16 | Read the xref:usage-guide/deep-storage.adoc[deep storage usage guide] to learn more about configuring Druid deep storage. 17 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/usage-guide/index.adoc: -------------------------------------------------------------------------------- 1 | = Usage guide 2 | :page-aliases: usage.doc 3 | 4 | The usage guide covers various aspects of configuring Druid and interconnection with other tools. 5 | 6 | xref:usage-guide/deep-storage.adoc[], xref:usage-guide/ingestion.adoc[] and xref:usage-guide/resources-and-storage.adoc[] are the relevant pages for configuring how your data is stored and ingested. 7 | 8 | The xref:usage-guide/security.adoc[] page explains how to configure TLS, authentication with LDAP and authorization using xref:opa:index.adoc[OPA]. 9 | 10 | Look into xref:usage-guide/logging.adoc[] and xref:usage-guide/monitoring.adoc[] to learn how to observe your Druid status. 11 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/usage-guide/ingestion.adoc: -------------------------------------------------------------------------------- 1 | = Ingestion 2 | :description: Ingest data from S3 by specifying the host and optional credentials. Add external files to Druid pods using extra volumes for client certificates or keytabs. 3 | 4 | == [[s3]]From S3 5 | 6 | To ingest data from s3 you need to specify a host to connect to, but there are also other settings that can be used: 7 | 8 | [source,yaml] 9 | ---- 10 | spec: 11 | clusterConfig: 12 | ingestion: 13 | s3connection: 14 | host: yourhost.com # <1> 15 | port: 80 # optional <2> 16 | credentials: # optional <3> 17 | ... 18 | ---- 19 | 20 | <1> The S3 host, not optional 21 | <2> Port, optional, defaults to 80 22 | <3> Credentials to use. 23 | Since these might be bucket-dependent, they can instead be given in the ingestion job. 24 | Specifying the credentials here is explained <>. 25 | 26 | include::partial$s3-note.adoc[] 27 | 28 | === S3 credentials 29 | 30 | include::partial$s3-credentials.adoc[] 31 | 32 | == Adding external files, e.g. for ingestion 33 | 34 | Since Druid actively runs ingestion tasks there may be a need to make extra files available to the processes. 35 | 36 | These could for example be client certificates used to connect to a Kafka cluster or a keytab to obtain a Kerberos ticket. 37 | 38 | In order to make these files available the operator allows specifying extra volumes that are added to all pods deployed for this cluster. 39 | 40 | [source,yaml] 41 | ---- 42 | spec: 43 | clusterConfig: 44 | extraVolumes: 45 | - name: google-service-account 46 | secret: 47 | secretName: google-service-account 48 | ---- 49 | 50 | All Volumes specified in this section are made available under `/stackable/userdata/\{volumename\}`. 51 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/usage-guide/listenerclass.adoc: -------------------------------------------------------------------------------- 1 | = Service exposition with ListenerClasses 2 | :description: Configure Apache Druid service exposure using ListenerClass to control service types: cluster-internal, external-unstable, or external-stable. 3 | 4 | Apache Druid offers a web UI and an API, both are exposed by the `router` role. 5 | Other roles also expose API endpoints such as the `broker` and `coordinator`. 6 | The Operator deploys services called `-` (where `` is the name of the DruidCluster and `` is the role for the Service) through which Druid can be reached. 7 | 8 | These services can have three different types: `cluster-internal`, `external-unstable` and `external-stable`. 9 | Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level. 10 | 11 | This is how the listener class is configured: 12 | 13 | [source,yaml] 14 | ---- 15 | spec: 16 | clusterConfig: 17 | listenerClass: cluster-internal # <1> 18 | ---- 19 | <1> The default `cluster-internal` setting. 20 | 21 | This setting affects all role Services at once. 22 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/usage-guide/logging.adoc: -------------------------------------------------------------------------------- 1 | = Log aggregation 2 | :description: Forward logs to a Vector aggregator by enabling the log agent and specifying a discovery ConfigMap. 3 | 4 | The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent: 5 | 6 | [source,yaml] 7 | ---- 8 | spec: 9 | clusterConfig: 10 | vectorAggregatorConfigMapName: vector-aggregator-discovery 11 | brokers: 12 | config: 13 | logging: 14 | enableVectorAgent: true 15 | coordinators: 16 | config: 17 | logging: 18 | enableVectorAgent: true 19 | historicals: 20 | config: 21 | logging: 22 | enableVectorAgent: true 23 | middleManagers: 24 | config: 25 | logging: 26 | enableVectorAgent: true 27 | routers: 28 | config: 29 | logging: 30 | enableVectorAgent: true 31 | ---- 32 | 33 | Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. 34 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/usage-guide/monitoring.adoc: -------------------------------------------------------------------------------- 1 | = Monitoring 2 | :description: Managed Druid instances export Prometheus metrics by default for easy monitoring. 3 | 4 | The managed Druid instances are automatically configured to export Prometheus metrics. 5 | See xref:operators:monitoring.adoc[] for more details. 6 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/usage-guide/operations/cluster-operations.adoc: -------------------------------------------------------------------------------- 1 | = Cluster operation 2 | 3 | Druid installations can be configured with different cluster operations like pausing reconciliation or stopping the cluster. 4 | See xref:concepts:operations/cluster_operations.adoc[cluster operations] for more details. 5 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/usage-guide/operations/index.adoc: -------------------------------------------------------------------------------- 1 | = Operations 2 | 3 | This section of the documentation is intended for the operations teams that maintain a Stackable Data Platform installation. 4 | 5 | Please read the xref:concepts:operations/index.adoc[Concepts page on Operations] that contains the necessary details to operate the platform in a production environment. 6 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/usage-guide/operations/pod-disruptions.adoc: -------------------------------------------------------------------------------- 1 | = Allowed Pod disruptions 2 | 3 | You can configure the permitted Pod disruptions for Druid nodes as described in xref:concepts:operations/pod_disruptions.adoc[]. 4 | 5 | Unless you configure something else or disable the default PodDisruptionBudgets (PDBs), the following PDBs apply: 6 | 7 | == Brokers 8 | Only one broker may be offline at any time, regardless of the number of replicas or role groups. 9 | 10 | == Coordinators 11 | Only one coordinator may be offline at any time, regardless of the number of replicas or role groups. 12 | 13 | == Historicals 14 | Only one historical may be offline at any time, regardless of the number of replicas or role groups. 15 | 16 | == MiddleManagers 17 | Only one middle manager may be offline at any time, regardless of the number of replicas or role groups. 18 | 19 | == Routers 20 | Only one router may be offline at any time, regardless of the number of replicas or role groups. 21 | -------------------------------------------------------------------------------- /docs/modules/druid/pages/usage-guide/operations/pod-placement.adoc: -------------------------------------------------------------------------------- 1 | = Pod placement 2 | 3 | You can configure the Pod placement of the Druid pods as described in xref:concepts:operations/pod_placement.adoc[]. 4 | 5 | The default affinities created by the operator are: 6 | 7 | 1. Distribute all Pods within the same role (brokers, coordinators, historicals, middle-managers, routers) (weight 70) 8 | 9 | Some of the Druid roles do frequently communicate with each other. 10 | To address this, some affinities are created to attract these roles: 11 | 12 | *For brokers:* 13 | 14 | 1. Co-locate with historicals (weight 60) 15 | 2. Co-locate with middle-managers (weight 40) 16 | 17 | *For routers:* 18 | 19 | 1. Co-locate with brokers (weight 40) 20 | 21 | *For historicals and middle-managers:* 22 | 23 | 1. Co-locate the middle-managers and historicals with the hdfs datanodes if hdfs is used as deep storage (weight 50) 24 | 25 | *For coordinators:* 26 | 27 | No affinities 28 | -------------------------------------------------------------------------------- /docs/modules/druid/partials/nav.adoc: -------------------------------------------------------------------------------- 1 | * xref:druid:getting_started/index.adoc[] 2 | ** xref:druid:getting_started/installation.adoc[] 3 | ** xref:druid:getting_started/first_steps.adoc[] 4 | * xref:druid:required-external-components.adoc[] 5 | * xref:druid:usage-guide/index.adoc[] 6 | ** xref:druid:usage-guide/listenerclass.adoc[] 7 | ** xref:druid:usage-guide/ingestion.adoc[] 8 | ** xref:druid:usage-guide/deep-storage.adoc[] 9 | ** xref:druid:usage-guide/resources-and-storage.adoc[] 10 | ** xref:druid:usage-guide/security.adoc[] 11 | ** xref:druid:usage-guide/logging.adoc[] 12 | ** xref:druid:usage-guide/monitoring.adoc[] 13 | ** xref:druid:usage-guide/extensions.adoc[] 14 | ** xref:druid:usage-guide/overrides.adoc[] 15 | ** xref:druid:usage-guide/operations/index.adoc[] 16 | *** xref:druid:usage-guide/operations/cluster-operations.adoc[] 17 | *** xref:druid:usage-guide/operations/pod-placement.adoc[] 18 | *** xref:druid:usage-guide/operations/pod-disruptions.adoc[] 19 | *** xref:druid:usage-guide/operations/graceful-shutdown.adoc[] 20 | * xref:druid:reference/index.adoc[] 21 | ** xref:druid:reference/crds.adoc[] 22 | *** {crd-docs}/druid.stackable.tech/druidcluster/v1alpha1/[DruidCluster {external-link-icon}^] 23 | ** xref:druid:reference/discovery.adoc[] 24 | ** xref:druid:reference/commandline-parameters.adoc[] 25 | ** xref:druid:reference/environment-variables.adoc[] 26 | -------------------------------------------------------------------------------- /docs/modules/druid/partials/s3-credentials.adoc: -------------------------------------------------------------------------------- 1 | No matter if a connection is specified inline or as a separate object, the credentials are always specified in the same way. 2 | You need a Secret containing the access key ID and secret access key, a SecretClass and then a reference to this SecretClass where you want to specify the credentials. 3 | 4 | The Secret: 5 | 6 | [source,yaml] 7 | ---- 8 | apiVersion: v1 9 | kind: Secret 10 | metadata: 11 | name: s3-credentials 12 | labels: 13 | secrets.stackable.tech/class: s3-credentials-class # <1> 14 | stringData: 15 | accessKey: YOUR_VALID_ACCESS_KEY_ID_HERE 16 | secretKey: YOUR_SECRET_ACCES_KEY_THATBELONGS_TO_THE_KEY_ID_HERE 17 | ---- 18 | 19 | <1> This label connects the `Secret` to the `SecretClass`. 20 | 21 | The `SecretClass`: 22 | 23 | [source,yaml] 24 | ---- 25 | apiVersion: secrets.stackable.tech/v1alpha1 26 | kind: SecretClass 27 | metadata: 28 | name: s3-credentials-class 29 | spec: 30 | backend: 31 | k8sSearch: 32 | searchNamespace: 33 | pod: {} 34 | ---- 35 | 36 | Referencing it: 37 | 38 | [source,yaml] 39 | ---- 40 | ... 41 | credentials: 42 | secretClass: s3-credentials-class 43 | ... 44 | ---- 45 | -------------------------------------------------------------------------------- /docs/modules/druid/partials/s3-note.adoc: -------------------------------------------------------------------------------- 1 | [NOTE] 2 | ==== 3 | You can specify just a connection/bucket for either ingestion or deep storage or for both, but Druid only supports a single S3 connection under the hood. 4 | If two connections are specified, they must be the same. This is easiest if a dedicated S3 Connection Resource is used - not defined inline but as a dedicated object. 5 | 6 | The `S3Connection` `region` field is ignored because Druid uses the AWS SDK v1, which ignores the region if the endpoint is set. 7 | The host is a required field, therefore the endpoint will always be set. 8 | 9 | TLS for S3 is not yet supported. 10 | ==== 11 | -------------------------------------------------------------------------------- /docs/modules/druid/partials/supported-versions.adoc: -------------------------------------------------------------------------------- 1 | // The version ranges supported by Druid-Operator 2 | // This is a separate file, since it is used by both the direct Druid documentation, and the overarching 3 | // Stackable Platform documentation. 4 | 5 | - 33.0.0 6 | - 31.0.1 (deprecated) 7 | - 30.0.1 (LTS) 8 | -------------------------------------------------------------------------------- /docs/templating_vars.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | helm: 3 | repo_name: sdp-charts 4 | repo_url: oci.stackable.tech 5 | versions: 6 | commons: 0.0.0-dev 7 | secret: 0.0.0-dev 8 | listener: 0.0.0-dev 9 | zookeeper: 0.0.0-dev 10 | hdfs: 0.0.0-dev 11 | druid: 0.0.0-dev 12 | postgresql: 16.5.0 13 | -------------------------------------------------------------------------------- /examples/psql-s3/README.md: -------------------------------------------------------------------------------- 1 | A PostgreSQL database is required, you can spin up a PostgreSQL database with the bitnami PostgreSQL helm chart. 2 | Add the bitname repository: 3 | 4 | helm repo add bitnami https://charts.bitnami.com/bitnami 5 | 6 | And setup the Postgres database: 7 | 8 | helm install druid bitnami/postgresql \ 9 | --version=11 \ 10 | --set auth.username=druid \ 11 | --set auth.password=druid \ 12 | --set auth.database=druid 13 | 14 | Make sure to adapt the S3 secret in `./druid-cluster.yaml` with your credentials: 15 | 16 | apiVersion: v1 17 | kind: Secret 18 | metadata: 19 | name: s3-credentials 20 | stringData: 21 | accessKeyId: YOUR_VALID_ACCESS_KEY_ID_HERE 22 | secretAccessKey: YOUR_SECRET_ACCES_KEY_THATBELONGS_TO_THE_KEY_ID_HERE 23 | -------------------------------------------------------------------------------- /examples/psql/README.md: -------------------------------------------------------------------------------- 1 | A PostgreSQL database is required, you can spin up a PostgreSQL database with the bitnami PostgreSQL helm chart. 2 | Add the bitname repository: 3 | 4 | helm repo add bitnami https://charts.bitnami.com/bitnami 5 | 6 | And setup the Postgres database: 7 | 8 | helm install druid bitnami/postgresql \ 9 | --version=11 \ 10 | --set auth.username=druid \ 11 | --set auth.password=druid \ 12 | --set auth.database=druid 13 | -------------------------------------------------------------------------------- /nix/README.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | # Updating nix dependencies 7 | 8 | ## Run the following for an operator 9 | 10 | > [!NOTE] 11 | > We track the `master` branch of crate2nix as that is relatively up to date, but the releases are infrequent. 12 | 13 | ```shell 14 | niv update crate2nix 15 | niv update nixpkgs 16 | niv update beku.py -b X.Y.Z # Using the release tag 17 | ``` 18 | 19 | ### Test 20 | 21 | - Run make `regenerate-nix` to ensure crate2nix works 22 | - Run a smoke test to ensure beku.py works. 23 | - Run `make run-dev` to ensure nixpkgs are fine. 24 | 25 | ## Update operator-templating 26 | 27 | Do the same as above, but from `template/` 28 | -------------------------------------------------------------------------------- /nix/meta.json: -------------------------------------------------------------------------------- 1 | {"operator": {"name": "druid-operator", "pretty_string": "Apache Druid", "product_string": "druid", "url": "stackabletech/druid-operator.git"}} 2 | -------------------------------------------------------------------------------- /nix/sources.json: -------------------------------------------------------------------------------- 1 | { 2 | "beku.py": { 3 | "branch": "0.0.10", 4 | "description": "Test suite expander for Stackable Kuttl tests.", 5 | "homepage": null, 6 | "owner": "stackabletech", 7 | "repo": "beku.py", 8 | "rev": "fc75202a38529a4ac6776dd8a5dfee278d927f58", 9 | "sha256": "152yary0p11h87yabv74jnwkghsal7lx16az0qlzrzdrs6n5v8id", 10 | "type": "tarball", 11 | "url": "https://github.com/stackabletech/beku.py/archive/fc75202a38529a4ac6776dd8a5dfee278d927f58.tar.gz", 12 | "url_template": "https://github.com///archive/.tar.gz" 13 | }, 14 | "crate2nix": { 15 | "branch": "master", 16 | "description": "nix build file generator for rust crates", 17 | "homepage": "", 18 | "owner": "kolloch", 19 | "repo": "crate2nix", 20 | "rev": "be31feae9a82c225c0fd1bdf978565dc452a483a", 21 | "sha256": "14d0ymlrwk7dynv35qcw4xn0dylfpwjmf6f8znflbk2l6fk23l12", 22 | "type": "tarball", 23 | "url": "https://github.com/kolloch/crate2nix/archive/be31feae9a82c225c0fd1bdf978565dc452a483a.tar.gz", 24 | "url_template": "https://github.com///archive/.tar.gz" 25 | }, 26 | "nixpkgs": { 27 | "branch": "nixpkgs-unstable", 28 | "description": "Nix Packages collection", 29 | "homepage": "", 30 | "owner": "NixOS", 31 | "repo": "nixpkgs", 32 | "rev": "b1bebd0fe266bbd1820019612ead889e96a8fa2d", 33 | "sha256": "0fl2dji5whjydbxby9b7kqyqx9m4k44p72x1q28kfnx5m67nyqij", 34 | "type": "tarball", 35 | "url": "https://github.com/NixOS/nixpkgs/archive/b1bebd0fe266bbd1820019612ead889e96a8fa2d.tar.gz", 36 | "url_template": "https://github.com///archive/.tar.gz" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "local>stackabletech/.github:renovate-config" 5 | ], 6 | "ignorePaths": [".github/workflows/build.yml", ".github/workflows/general_daily_security.yml", ".github/workflows/integration-test.yml", ".github/workflows/pr_pre-commit.yaml"] 7 | } 8 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | # DO NOT EDIT, this file is generated by operator-templating 2 | [toolchain] 3 | channel = "1.85.0" 4 | profile = "default" 5 | -------------------------------------------------------------------------------- /rust/operator-binary/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "stackable-druid-operator" 3 | description = "Stackable Operator for Apache Druid" 4 | version.workspace = true 5 | authors.workspace = true 6 | license.workspace = true 7 | edition.workspace = true 8 | repository.workspace = true 9 | publish = false 10 | 11 | [dependencies] 12 | product-config.workspace = true 13 | stackable-operator.workspace = true 14 | 15 | anyhow.workspace = true 16 | clap.workspace = true 17 | const_format.workspace = true 18 | fnv.workspace = true 19 | futures.workspace = true 20 | indoc.workspace = true 21 | openssl.workspace = true 22 | pin-project.workspace = true 23 | semver.workspace = true 24 | serde_json.workspace = true 25 | serde.workspace = true 26 | snafu.workspace = true 27 | strum.workspace = true 28 | tokio.workspace = true 29 | tracing.workspace = true 30 | 31 | [build-dependencies] 32 | built.workspace = true 33 | 34 | [dev-dependencies] 35 | rstest.workspace = true 36 | serde_yaml.workspace = true 37 | -------------------------------------------------------------------------------- /rust/operator-binary/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | built::write_built_file().unwrap(); 3 | } 4 | -------------------------------------------------------------------------------- /rust/operator-binary/src/config/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod jvm; 2 | -------------------------------------------------------------------------------- /rust/operator-binary/src/crd/authorization.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use stackable_operator::{ 3 | commons::opa::OpaConfig, 4 | schemars::{self, JsonSchema}, 5 | }; 6 | 7 | #[derive(Clone, Deserialize, Debug, Default, Eq, JsonSchema, PartialEq, Serialize)] 8 | #[serde(rename_all = "camelCase")] 9 | pub struct DruidAuthorization { 10 | /// Configure the OPA stacklet [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery) 11 | /// and the name of the Rego package containing your Druid authorization rules. 12 | /// Consult the [OPA authorization documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/opa) 13 | /// to learn how to deploy Rego authorization rules with OPA. 14 | /// Read the [Druid operator security documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/security) 15 | /// for more information on how to write rules specifically for Druid. 16 | pub opa: OpaConfig, 17 | } 18 | -------------------------------------------------------------------------------- /rust/operator-binary/src/operations/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod graceful_shutdown; 2 | pub mod pdb; 3 | -------------------------------------------------------------------------------- /rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: druid.stackable.tech/v1alpha1 3 | kind: DruidCluster 4 | metadata: 5 | name: test-resource-merge 6 | namespace: default 7 | uid: test-resource-merge 8 | spec: 9 | image: 10 | productVersion: 30.0.0 11 | clusterConfig: 12 | deepStorage: 13 | hdfs: 14 | configMapName: simple-hdfs 15 | directory: /druid 16 | metadataStorageDatabase: 17 | dbType: postgresql 18 | connString: jdbc:postgresql://druid-postgresql/druid 19 | host: druid-postgresql 20 | port: 5432 21 | credentialsSecret: mySecret 22 | zookeeperConfigMapName: psql-druid-znode 23 | brokers: 24 | roleGroups: 25 | default: 26 | replicas: 1 27 | coordinators: 28 | roleGroups: 29 | default: 30 | replicas: 1 31 | historicals: 32 | roleGroups: 33 | default: 34 | replicas: 1 35 | middleManagers: 36 | config: 37 | resources: 38 | cpu: 39 | min: 100m 40 | max: "1" 41 | memory: 42 | limit: 1Gi 43 | roleGroups: 44 | resources-from-role: 45 | replicas: 1 46 | resources-from-role-group: 47 | config: 48 | resources: 49 | cpu: 50 | min: 300m 51 | max: "3" 52 | memory: 53 | limit: 3Gi 54 | replicas: 1 55 | routers: 56 | roleGroups: 57 | default: 58 | replicas: 1 59 | -------------------------------------------------------------------------------- /rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: druid.stackable.tech/v1alpha1 3 | kind: DruidCluster 4 | metadata: 5 | name: test-resource-merge 6 | namespace: default 7 | uid: test-resource-merge 8 | spec: 9 | image: 10 | productVersion: 30.0.0 11 | clusterConfig: 12 | deepStorage: 13 | hdfs: 14 | configMapName: simple-hdfs 15 | directory: /druid 16 | metadataStorageDatabase: 17 | dbType: postgresql 18 | connString: jdbc:postgresql://druid-postgresql/druid 19 | host: druid-postgresql 20 | port: 5432 21 | credentialsSecret: mySecret 22 | zookeeperConfigMapName: psql-druid-znode 23 | brokers: 24 | roleGroups: 25 | default: 26 | replicas: 1 27 | coordinators: 28 | roleGroups: 29 | default: 30 | replicas: 1 31 | historicals: 32 | config: 33 | resources: 34 | cpu: 35 | min: 100m 36 | max: "1" 37 | memory: 38 | limit: 1Gi 39 | storage: 40 | segmentCache: 41 | freePercentage: 3 42 | emptyDir: 43 | capacity: 5g 44 | roleGroups: 45 | default: 46 | replicas: 1 47 | secondary: 48 | replicas: 1 49 | config: 50 | resources: 51 | storage: 52 | segmentCache: 53 | freePercentage: 7 54 | emptyDir: 55 | capacity: 2g 56 | middleManagers: 57 | roleGroups: 58 | default: 59 | replicas: 1 60 | routers: 61 | roleGroups: 62 | default: 63 | replicas: 1 64 | -------------------------------------------------------------------------------- /rust/operator-binary/test/resources/crd/role_service/druid_cluster.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: druid.stackable.tech/v1alpha1 3 | kind: DruidCluster 4 | metadata: 5 | name: testcluster 6 | namespace: default 7 | uid: test-uid 8 | spec: 9 | image: 10 | productVersion: 30.0.0 11 | clusterConfig: 12 | deepStorage: 13 | hdfs: 14 | configMapName: simple-hdfs 15 | directory: /druid 16 | metadataStorageDatabase: 17 | dbType: postgresql 18 | connString: jdbc:postgresql://druid-postgresql/druid 19 | host: druid-postgresql 20 | port: 5432 21 | credentialsSecret: mySecret 22 | zookeeperConfigMapName: psql-druid-znode 23 | brokers: 24 | roleGroups: 25 | default: 26 | replicas: 1 27 | coordinators: 28 | roleGroups: 29 | default: 30 | replicas: 1 31 | historicals: 32 | roleGroups: 33 | default: 34 | replicas: 1 35 | middleManagers: 36 | roleGroups: 37 | default: 38 | replicas: 1 39 | routers: 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | -------------------------------------------------------------------------------- /rust/operator-binary/test/resources/druid_controller/segment_cache.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: druid.stackable.tech/v1alpha1 3 | kind: DruidCluster 4 | metadata: 5 | name: test-resource-merge 6 | namespace: default 7 | uid: test-resource-merge 8 | spec: 9 | image: 10 | productVersion: 30.0.0 11 | clusterConfig: 12 | deepStorage: 13 | s3: 14 | bucket: 15 | inline: 16 | bucketName: druid 17 | connection: 18 | inline: 19 | host: minio-druid 20 | port: 9000 21 | accessStyle: Path 22 | credentials: 23 | secretClass: druid-s3-credentials 24 | metadataStorageDatabase: 25 | dbType: postgresql 26 | connString: jdbc:postgresql://druid-postgresql/druid 27 | host: druid-postgresql 28 | port: 5432 29 | credentialsSecret: mySecret 30 | zookeeperConfigMapName: psql-druid-znode 31 | brokers: 32 | roleGroups: 33 | default: 34 | replicas: 1 35 | coordinators: 36 | roleGroups: 37 | default: 38 | replicas: 1 39 | historicals: 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | secondary: 44 | replicas: 1 45 | config: 46 | resources: 47 | storage: 48 | segmentCache: 49 | freePercentage: 2 50 | emptyDir: 51 | capacity: 5G 52 | middleManagers: 53 | roleGroups: 54 | default: 55 | replicas: 1 56 | routers: 57 | roleGroups: 58 | default: 59 | replicas: 1 60 | -------------------------------------------------------------------------------- /rust/operator-binary/test/resources/druid_controller/simple.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: druid.stackable.tech/v1alpha1 3 | kind: DruidCluster 4 | metadata: 5 | name: simple-druid 6 | namespace: default 7 | uid: test-uid 8 | spec: 9 | image: 10 | productVersion: 30.0.0 11 | clusterConfig: 12 | deepStorage: 13 | hdfs: 14 | configMapName: simple-hdfs 15 | directory: /druid 16 | metadataStorageDatabase: 17 | dbType: postgresql 18 | connString: jdbc:postgresql://druid-postgresql/druid 19 | host: druid-postgresql 20 | port: 5432 21 | credentialsSecret: mySecret 22 | zookeeperConfigMapName: psql-druid-znode 23 | additionalExtensions: 24 | - druid-avro-extensions 25 | - druid-azure-extensions 26 | - druid-histogram 27 | - druid-avro-extensions 28 | brokers: 29 | roleGroups: 30 | default: 31 | replicas: 1 32 | coordinators: 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | historicals: 37 | roleGroups: 38 | default: 39 | replicas: 1 40 | middleManagers: 41 | roleGroups: 42 | default: 43 | replicas: 1 44 | routers: 45 | roleGroups: 46 | default: 47 | replicas: 1 48 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # This file includes unstable features, so you need to run "cargo +nightly fmt" to format your code. 2 | # It's also ok to use the stable toolchain by simple running "cargo fmt", but using the nigthly formatter is prefered. 3 | 4 | # https://doc.rust-lang.org/nightly/edition-guide/rust-2024/rustfmt-style-edition.html 5 | style_edition = "2024" 6 | imports_granularity = "Crate" 7 | group_imports = "StdExternalCrate" 8 | reorder_impl_items = true 9 | use_field_init_shorthand = true 10 | -------------------------------------------------------------------------------- /scripts/docs_templating.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Reads a file with variables to insert into templates, and templates all .*.j2 files 5 | # in the 'docs' directory. 6 | # 7 | # dependencies 8 | # pip install jinja2-cli 9 | 10 | docs_dir="$(dirname "$0")/../docs" 11 | templating_vars_file="$docs_dir/templating_vars.yaml" 12 | 13 | # Check if files need templating 14 | if [[ -z $(find "$docs_dir" -name '*.j2') ]]; 15 | then 16 | echo "No files need templating, exiting." 17 | exit 18 | fi 19 | 20 | # Check if jinja2 is there 21 | if ! command -v jinja2 &> /dev/null 22 | then 23 | echo "jinja2 could not be found. Use 'pip install jinja2-cli' to install it." 24 | exit 1 25 | fi 26 | 27 | # Check if templating vars file exists 28 | if [[ ! -f "$templating_vars_file" ]]; 29 | then 30 | echo "$templating_vars_file does not exist, cannot start templating." 31 | fi 32 | 33 | find "$docs_dir" -name '*.j2' | 34 | while read -r file 35 | do 36 | new_file_name=${file%.j2} # Remove .j2 suffix 37 | echo "templating $new_file_name" 38 | jinja2 "$file" "$templating_vars_file" -o "$new_file_name" 39 | done 40 | 41 | echo "done" 42 | -------------------------------------------------------------------------------- /scripts/generate-manifests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This script reads a Helm chart from deploy/helm/druid-operator and 3 | # generates manifest files into deploy/manifestss 4 | set -e 5 | 6 | tmp=$(mktemp -d ./manifests-XXXXX) 7 | 8 | helm template --output-dir "$tmp" \ 9 | --include-crds \ 10 | --name-template druid-operator \ 11 | deploy/helm/druid-operator 12 | 13 | for file in "$tmp"/druid-operator/*/*; do 14 | yq eval -i 'del(.. | select(has("app.kubernetes.io/managed-by")) | ."app.kubernetes.io/managed-by")' /dev/stdin < "$file" 15 | yq eval -i 'del(.. | select(has("helm.sh/chart")) | ."helm.sh/chart")' /dev/stdin < "$file" 16 | sed -i '/# Source: .*/d' "$file" 17 | done 18 | 19 | cp -r "$tmp"/druid-operator/*/* deploy/manifests/ 20 | 21 | rm -rf "$tmp" 22 | -------------------------------------------------------------------------------- /scripts/render_readme.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Check if jinja2 is there 5 | if ! command -v jinja2 &> /dev/null 6 | then 7 | echo "jinja2 could not be found. Use 'pip install jinja2-cli' to install it." 8 | exit 1 9 | fi 10 | 11 | SCRIPT_DIR=$(dirname "$0") 12 | cd "$SCRIPT_DIR/../.readme" 13 | jinja2 README.md.j2 -o ../README.md 14 | cd .. 15 | 16 | python3 scripts/ensure_one_trailing_newline.py README.md 17 | -------------------------------------------------------------------------------- /scripts/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ./scripts/run-tests "$@" 4 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | let 2 | self = import ./. {}; 3 | inherit (self) sources pkgs meta; 4 | 5 | beku = pkgs.callPackage (sources."beku.py" + "/beku.nix") {}; 6 | cargoDependencySetOfCrate = crate: [ crate ] ++ pkgs.lib.concatMap cargoDependencySetOfCrate (crate.dependencies ++ crate.buildDependencies); 7 | cargoDependencySet = pkgs.lib.unique (pkgs.lib.flatten (pkgs.lib.mapAttrsToList (crateName: crate: cargoDependencySetOfCrate crate.build) self.cargo.workspaceMembers)); 8 | in pkgs.mkShell rec { 9 | name = meta.operator.name; 10 | 11 | packages = with pkgs; [ 12 | ## cargo et-al 13 | rustup # this breaks pkg-config if it is in the nativeBuildInputs 14 | cargo-udeps 15 | 16 | ## Extra dependencies for use in a pure env (nix-shell --pure) 17 | ## These are mosuly useful for maintainers of this shell.nix 18 | ## to ensure all the dependencies are caught. 19 | # cacert 20 | # vim nvim nano 21 | ]; 22 | 23 | # derivation runtime dependencies 24 | buildInputs = pkgs.lib.concatMap (crate: crate.buildInputs) cargoDependencySet; 25 | 26 | # build time dependencies 27 | nativeBuildInputs = pkgs.lib.concatMap (crate: crate.nativeBuildInputs) cargoDependencySet ++ (with pkgs; [ 28 | beku 29 | docker 30 | gettext # for the proper envsubst 31 | git 32 | jq 33 | kind 34 | kubectl 35 | kubernetes-helm 36 | kuttl 37 | nix # this is implied, but needed in the pure env 38 | # tilt already defined in default.nix 39 | which 40 | yq-go 41 | ]); 42 | 43 | LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; 44 | BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include"; 45 | } 46 | -------------------------------------------------------------------------------- /tests/infrastructure.yaml: -------------------------------------------------------------------------------- 1 | instance-size: medium 2 | disk: 100 3 | nodes: 5 4 | parallelism: 1 5 | -------------------------------------------------------------------------------- /tests/kuttl-test.yaml.jinja2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestSuite 4 | testDirs: 5 | {% for testcase in testinput.tests %} 6 | - ./tests/{{ testcase.name }} 7 | {% endfor %} 8 | 9 | startKIND: false 10 | suppress: ["events"] 11 | parallel: 2 12 | 13 | # The timeout (in seconds) is used when namespaces are created or 14 | # deleted, and, if not overridden, in TestSteps, TestAsserts, and 15 | # Commands. If not set, the timeout is 30 seconds by default. 16 | # 17 | # The deletion of a namespace can take a while until all resources, 18 | # especially PersistentVolumeClaims, are gracefully shut down. If the 19 | # timeout is reached in the meantime, even a successful test case is 20 | # considered a failure. 21 | # 22 | # For instance, the termination grace period of the Vector aggregator in 23 | # the logging tests is set to 60 seconds. If there are logs entries 24 | # which could not be forwarded yet to the external aggregator defined in 25 | # the VECTOR_AGGREGATOR environment variable, then the test aggregator 26 | # uses this period of time by trying to forward the events. In this 27 | # case, deleting a namespace with several Pods takes about 90 seconds. 28 | timeout: 300 29 | -------------------------------------------------------------------------------- /tests/release.yaml: -------------------------------------------------------------------------------- 1 | # Contains all operators required to run the test suite. 2 | --- 3 | releases: 4 | # Do not change the name of the release as it's referenced from run-tests 5 | tests: 6 | releaseDate: 1970-01-01 7 | description: Integration test 8 | products: 9 | commons: 10 | operatorVersion: 0.0.0-dev 11 | secret: 12 | operatorVersion: 0.0.0-dev 13 | listener: 14 | operatorVersion: 0.0.0-dev 15 | zookeeper: 16 | operatorVersion: 0.0.0-dev 17 | hdfs: 18 | operatorVersion: 0.0.0-dev 19 | druid: 20 | operatorVersion: 0.0.0-dev 21 | opa: 22 | operatorVersion: 0.0.0-dev 23 | -------------------------------------------------------------------------------- /tests/templates/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/druid-operator/26e58068bbf7ed13cb89b70bf474336dd7190097/tests/templates/.gitkeep -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: hdfs-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/01-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: druid-zk 29 | --- 30 | apiVersion: zookeeper.stackable.tech/v1alpha1 31 | kind: ZookeeperZnode 32 | metadata: 33 | name: hdfs-znode 34 | spec: 35 | clusterRef: 36 | name: druid-zk 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl -n $NAMESPACE rollout status daemonset test-opa-server-default --timeout 300s 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/02-install-opa.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: test-rego 6 | labels: 7 | opa.stackable.tech/bundle: "true" 8 | data: 9 | test.rego: | 10 | package druid 11 | 12 | default allow = false 13 | 14 | allow if { 15 | input.authenticationResult.identity == "alice" 16 | } 17 | 18 | allow if { 19 | input.authenticationResult.identity == "admin" 20 | } 21 | 22 | allow if { 23 | input.authenticationResult.identity == "druid_system" 24 | } 25 | --- 26 | apiVersion: opa.stackable.tech/v1alpha1 27 | kind: OpaCluster 28 | metadata: 29 | name: test-opa 30 | spec: 31 | image: 32 | {% if test_scenario['values']['opa'].find(",") > 0 %} 33 | custom: "{{ test_scenario['values']['opa'].split(',')[1] }}" 34 | productVersion: "{{ test_scenario['values']['opa'].split(',')[0] }}" 35 | {% else %} 36 | productVersion: "{{ test_scenario['values']['opa'] }}" 37 | {% endif %} 38 | pullPolicy: IfNotPresent 39 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 40 | clusterConfig: 41 | vectorAggregatorConfigMapName: vector-aggregator-discovery 42 | {% endif %} 43 | servers: 44 | roleGroups: 45 | default: 46 | replicas: 1 47 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-hdfs-namenode-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-hdfs-journalnode-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-hdfs-datanode-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/03-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: druid-hdfs 6 | timeout: 600 7 | --- 8 | apiVersion: hdfs.stackable.tech/v1alpha1 9 | kind: HdfsCluster 10 | metadata: 11 | name: druid-hdfs 12 | spec: 13 | image: 14 | productVersion: "{{ test_scenario['values']['hadoop'] }}" 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | dfsReplication: 1 18 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 19 | vectorAggregatorConfigMapName: vector-aggregator-discovery 20 | {% endif %} 21 | zookeeperConfigMapName: hdfs-znode 22 | nameNodes: 23 | config: 24 | logging: 25 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 26 | roleGroups: 27 | default: 28 | replicas: 2 29 | dataNodes: 30 | config: 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | journalNodes: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/04-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: derby-druid-broker-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: derby-druid-coordinator-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: derby-druid-historical-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | --- 30 | apiVersion: apps/v1 31 | kind: StatefulSet 32 | metadata: 33 | name: derby-druid-middlemanager-default 34 | status: 35 | readyReplicas: 1 36 | replicas: 1 37 | --- 38 | apiVersion: apps/v1 39 | kind: StatefulSet 40 | metadata: 41 | name: derby-druid-router-default 42 | status: 43 | readyReplicas: 1 44 | replicas: 1 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: checks 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/05-checks-container.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: checks 6 | labels: 7 | app: checks 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: checks 13 | template: 14 | metadata: 15 | labels: 16 | app: checks 17 | spec: 18 | containers: 19 | - name: checks 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | command: ["sleep", "infinity"] 22 | terminationGracePeriodSeconds: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/06-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/authcheck.py derby-druid 6 | timeout: 600 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/06-authcheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ./authcheck.py checks-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/authorizer/README.md: -------------------------------------------------------------------------------- 1 | # Authorizer Test 2 | 3 | Required Operators: 4 | 5 | - Zookeeper 6 | - Druid 7 | - RegoRule 8 | - OPA 9 | 10 | 1. Deploy Zookeeper Cluster 11 | 2. Deploy OPA Cluster + RegoRule 12 | 3. Deploy Druid Cluster 13 | 4. Setup Test Container 14 | 5. Run Auth Test: 15 | - Create two test users: Alice, Eve 16 | - Run HTTP requests to test if authentication + authorization works: 17 | - unauthenticated user should get a 401 18 | - unauthorized user should get a 403 19 | - authorized user should get the "normal" response 20 | 21 | To run only this test, use `kuttl test tests/opa` 22 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: druid-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/10-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: druid-zk 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-hdfs-namenode-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-hdfs-journalnode-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-hdfs-datanode-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/20-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | --- 6 | apiVersion: hdfs.stackable.tech/v1alpha1 7 | kind: HdfsCluster 8 | metadata: 9 | name: druid-hdfs 10 | spec: 11 | image: 12 | productVersion: "{{ test_scenario['values']['hadoop-latest'] }}" 13 | pullPolicy: IfNotPresent 14 | clusterConfig: 15 | dfsReplication: 1 16 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 17 | vectorAggregatorConfigMapName: vector-aggregator-discovery 18 | {% endif %} 19 | zookeeperConfigMapName: druid-hdfs-znode 20 | nameNodes: 21 | config: 22 | logging: 23 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 24 | roleGroups: 25 | default: 26 | replicas: 2 27 | dataNodes: 28 | config: 29 | logging: 30 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 31 | roleGroups: 32 | default: 33 | replicas: 1 34 | journalNodes: 35 | config: 36 | logging: 37 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 38 | roleGroups: 39 | default: 40 | replicas: 1 41 | --- 42 | apiVersion: zookeeper.stackable.tech/v1alpha1 43 | kind: ZookeeperZnode 44 | metadata: 45 | name: druid-hdfs-znode 46 | spec: 47 | clusterRef: 48 | name: druid-zk 49 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | commands: 6 | - script: kubectl -n $NAMESPACE wait --for=condition=available druidclusters.druid.stackable.tech/derby-druid --timeout 601s 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: derby-druid-broker-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: derby-druid-coordinator-default 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: derby-druid-historical-default 28 | status: 29 | readyReplicas: 1 30 | replicas: 1 31 | --- 32 | apiVersion: apps/v1 33 | kind: StatefulSet 34 | metadata: 35 | name: derby-druid-middlemanager-default 36 | status: 37 | readyReplicas: 1 38 | replicas: 1 39 | --- 40 | apiVersion: apps/v1 41 | kind: StatefulSet 42 | metadata: 43 | name: derby-druid-router-default 44 | status: 45 | readyReplicas: 1 46 | replicas: 1 47 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 180 5 | commands: 6 | - script: kubectl -n $NAMESPACE wait --for=condition=stopped druidclusters.druid.stackable.tech/derby-druid --timeout 181s 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: derby-druid-broker-default 12 | status: 13 | availableReplicas: 0 14 | replicas: 0 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: derby-druid-coordinator-default 20 | status: 21 | availableReplicas: 0 22 | replicas: 0 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: derby-druid-historical-default 28 | status: 29 | availableReplicas: 0 30 | replicas: 0 31 | --- 32 | apiVersion: apps/v1 33 | kind: StatefulSet 34 | metadata: 35 | name: derby-druid-middlemanager-default 36 | status: 37 | availableReplicas: 0 38 | replicas: 0 39 | --- 40 | apiVersion: apps/v1 41 | kind: StatefulSet 42 | metadata: 43 | name: derby-druid-router-default 44 | status: 45 | availableReplicas: 0 46 | replicas: 0 47 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 180 5 | commands: 6 | - script: kubectl -n $NAMESPACE wait --for=condition=reconciliationPaused druidclusters.druid.stackable.tech/derby-druid --timeout 181s 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: derby-druid-broker-default 12 | status: 13 | availableReplicas: 0 14 | replicas: 0 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: derby-druid-coordinator-default 20 | status: 21 | availableReplicas: 0 22 | replicas: 0 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: derby-druid-historical-default 28 | status: 29 | availableReplicas: 0 30 | replicas: 0 31 | --- 32 | apiVersion: apps/v1 33 | kind: StatefulSet 34 | metadata: 35 | name: derby-druid-middlemanager-default 36 | status: 37 | availableReplicas: 0 38 | replicas: 0 39 | --- 40 | apiVersion: apps/v1 41 | kind: StatefulSet 42 | metadata: 43 | name: derby-druid-router-default 44 | status: 45 | availableReplicas: 0 46 | replicas: 0 47 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | commands: 6 | - script: kubectl -n $NAMESPACE wait --for=condition=available druidclusters.druid.stackable.tech/derby-druid --timeout 601s 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: derby-druid-broker-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: derby-druid-coordinator-default 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: derby-druid-historical-default 28 | status: 29 | readyReplicas: 1 30 | replicas: 1 31 | --- 32 | apiVersion: apps/v1 33 | kind: StatefulSet 34 | metadata: 35 | name: derby-druid-middlemanager-default 36 | status: 37 | readyReplicas: 1 38 | replicas: 1 39 | --- 40 | apiVersion: apps/v1 41 | kind: StatefulSet 42 | metadata: 43 | name: derby-druid-router-default 44 | status: 45 | readyReplicas: 1 46 | replicas: 1 47 | -------------------------------------------------------------------------------- /tests/templates/kuttl/commons/druid-quickstartimport.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "index_parallel", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "index_parallel", 6 | "inputSource": { 7 | "type": "local", 8 | "baseDir": "quickstart/tutorial/", 9 | "filter": "wikiticker-2015-09-12-sampled.json.gz" 10 | }, 11 | "inputFormat": { 12 | "type": "json" 13 | } 14 | }, 15 | "tuningConfig": { 16 | "type": "index_parallel", 17 | "partitionsSpec": { 18 | "type": "dynamic" 19 | } 20 | }, 21 | "dataSchema": { 22 | "dataSource": "wikipedia-2015-09-12", 23 | "timestampSpec": { 24 | "column": "time", 25 | "format": "iso" 26 | }, 27 | "dimensionsSpec": { 28 | "dimensions": [ 29 | "channel", 30 | "cityName", 31 | "comment", 32 | "countryIsoCode", 33 | "countryName", 34 | "isAnonymous", 35 | "isMinor", 36 | "isNew", 37 | "isRobot", 38 | "isUnpatrolled", 39 | "metroCode", 40 | "namespace", 41 | "page", 42 | "regionIsoCode", 43 | "regionName", 44 | "user", 45 | { 46 | "type": "long", 47 | "name": "delta" 48 | }, 49 | { 50 | "type": "long", 51 | "name": "added" 52 | }, 53 | { 54 | "type": "long", 55 | "name": "deleted" 56 | } 57 | ] 58 | }, 59 | "granularitySpec": { 60 | "queryGranularity": "none", 61 | "rollup": false, 62 | "segmentGranularity": "day" 63 | } 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: druid-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/01-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: druid-zk 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-hdfs-namenode-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-hdfs-journalnode-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-hdfs-datanode-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/02-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: druid-hdfs 6 | timeout: 600 7 | --- 8 | apiVersion: hdfs.stackable.tech/v1alpha1 9 | kind: HdfsCluster 10 | metadata: 11 | name: druid-hdfs 12 | spec: 13 | image: 14 | productVersion: "{{ test_scenario['values']['hadoop'] }}" 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | dfsReplication: 1 18 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 19 | vectorAggregatorConfigMapName: vector-aggregator-discovery 20 | {% endif %} 21 | zookeeperConfigMapName: druid-hdfs-znode 22 | nameNodes: 23 | config: 24 | logging: 25 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 26 | roleGroups: 27 | default: 28 | replicas: 2 29 | dataNodes: 30 | config: 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | journalNodes: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | --- 44 | apiVersion: zookeeper.stackable.tech/v1alpha1 45 | kind: ZookeeperZnode 46 | metadata: 47 | name: druid-hdfs-znode 48 | spec: 49 | clusterRef: 50 | name: druid-zk 51 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: derby-druid-broker-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: derby-druid-coordinator-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: derby-druid-historical-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | --- 30 | apiVersion: apps/v1 31 | kind: StatefulSet 32 | metadata: 33 | name: derby-druid-middlemanager-default 34 | status: 35 | readyReplicas: 1 36 | replicas: 1 37 | --- 38 | apiVersion: apps/v1 39 | kind: StatefulSet 40 | metadata: 41 | name: derby-druid-router-default 42 | status: 43 | readyReplicas: 1 44 | replicas: 1 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/04-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: checks 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/04-checks-container.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: checks 6 | labels: 7 | app: checks 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: checks 13 | template: 14 | metadata: 15 | labels: 16 | app: checks 17 | spec: 18 | containers: 19 | - name: checks 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | command: ["sleep", "infinity"] 22 | terminationGracePeriodSeconds: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/healthcheck.py derby-druid 6 | timeout: 300 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/05-healthcheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/healthcheck.py checks-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/06-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/ingestioncheck.py derby-druid 6 | timeout: 300 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/06-ingestioncheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/ingestioncheck.py checks-0:/tmp 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/druid-quickstartimport.json checks-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/hdfs-deep-storage/07-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE druid-hdfs-namenode-default-0 -- ./bin/hdfs dfs -ls /druid | grep -q /druid/wikipedia-2015-09-12 6 | timeout: 30 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: hdfs-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/01-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: druid-zk 29 | --- 30 | apiVersion: zookeeper.stackable.tech/v1alpha1 31 | kind: ZookeeperZnode 32 | metadata: 33 | name: hdfs-znode 34 | spec: 35 | clusterRef: 36 | name: druid-zk 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-hdfs-namenode-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-hdfs-journalnode-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-hdfs-datanode-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/02-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: druid-hdfs 6 | timeout: 600 7 | --- 8 | apiVersion: hdfs.stackable.tech/v1alpha1 9 | kind: HdfsCluster 10 | metadata: 11 | name: druid-hdfs 12 | spec: 13 | image: 14 | productVersion: "{{ test_scenario['values']['hadoop'] }}" 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | dfsReplication: 1 18 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 19 | vectorAggregatorConfigMapName: vector-aggregator-discovery 20 | {% endif %} 21 | zookeeperConfigMapName: hdfs-znode 22 | nameNodes: 23 | config: 24 | logging: 25 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 26 | roleGroups: 27 | default: 28 | replicas: 2 29 | dataNodes: 30 | config: 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | journalNodes: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: derby-druid-broker-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: derby-druid-coordinator-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: derby-druid-historical-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | --- 30 | apiVersion: apps/v1 31 | kind: StatefulSet 32 | metadata: 33 | name: derby-druid-middlemanager-default 34 | status: 35 | readyReplicas: 1 36 | replicas: 1 37 | --- 38 | apiVersion: apps/v1 39 | kind: StatefulSet 40 | metadata: 41 | name: derby-druid-router-default 42 | status: 43 | readyReplicas: 1 44 | replicas: 1 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/04-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: checks 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/04-checks-container.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: checks 6 | labels: 7 | app: checks 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: checks 13 | template: 14 | metadata: 15 | labels: 16 | app: checks 17 | spec: 18 | containers: 19 | - name: checks 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | command: ["sleep", "infinity"] 22 | terminationGracePeriodSeconds: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/healthcheck.py derby-druid 6 | timeout: 300 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/05-healthcheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/healthcheck.py checks-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/06-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/ingestioncheck.py derby-druid 6 | timeout: 300 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-no-s3-ext/06-ingestioncheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/ingestioncheck.py checks-0:/tmp 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/druid-quickstartimport.json checks-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: hdfs-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/01-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: druid-zk 29 | --- 30 | apiVersion: zookeeper.stackable.tech/v1alpha1 31 | kind: ZookeeperZnode 32 | metadata: 33 | name: hdfs-znode 34 | spec: 35 | clusterRef: 36 | name: druid-zk 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-hdfs-namenode-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-hdfs-journalnode-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-hdfs-datanode-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/02-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: druid-hdfs 6 | timeout: 600 7 | --- 8 | apiVersion: hdfs.stackable.tech/v1alpha1 9 | kind: HdfsCluster 10 | metadata: 11 | name: druid-hdfs 12 | spec: 13 | image: 14 | productVersion: "{{ test_scenario['values']['hadoop'] }}" 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | dfsReplication: 1 18 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 19 | vectorAggregatorConfigMapName: vector-aggregator-discovery 20 | {% endif %} 21 | zookeeperConfigMapName: hdfs-znode 22 | nameNodes: 23 | config: 24 | logging: 25 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 26 | roleGroups: 27 | default: 28 | replicas: 2 29 | dataNodes: 30 | config: 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | journalNodes: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: derby-druid-broker-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: derby-druid-coordinator-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: derby-druid-historical-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | --- 30 | apiVersion: apps/v1 31 | kind: StatefulSet 32 | metadata: 33 | name: derby-druid-middlemanager-default 34 | status: 35 | readyReplicas: 1 36 | replicas: 1 37 | --- 38 | apiVersion: apps/v1 39 | kind: StatefulSet 40 | metadata: 41 | name: derby-druid-router-default 42 | status: 43 | readyReplicas: 1 44 | replicas: 1 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/04-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: checks 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/04-checks-container.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: checks 6 | labels: 7 | app: checks 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: checks 13 | template: 14 | metadata: 15 | labels: 16 | app: checks 17 | spec: 18 | containers: 19 | - name: checks 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | command: ["sleep", "infinity"] 22 | terminationGracePeriodSeconds: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/healthcheck.py derby-druid 6 | timeout: 300 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/05-healthcheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/healthcheck.py checks-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/06-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/ingestioncheck.py derby-druid 6 | timeout: 300 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ingestion-s3-ext/06-ingestioncheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/ingestioncheck.py checks-0:/tmp 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/druid-quickstartimport.json checks-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: openldap 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/01-install-openldap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | # We need to replace $NAMESPACE (by KUTTL) in the install-openldap.yaml 6 | - script: envsubst < 01_openldap.yaml | kubectl apply -n $NAMESPACE -f - 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE openldap-0 -- ldapsearch -H ldap://localhost:1389 -D "cn=integrationtest,ou=my users,dc=example,dc=org" -w 'bindPasswordWithSpecialCharacter\@<&>"'"'" -b "ou=my users,dc=example,dc=org" > /dev/null 6 | - script: kubectl exec -n $NAMESPACE openldap-0 -- bash -c LDAPTLS_CACERT=/tls/ca.crt ldapsearch -Z -H ldaps://localhost:1636 -D "cn=integrationtest,ou=my users,dc=example,dc=org" -w 'bindPasswordWithSpecialCharacter\@<&>"'"'" -b "ou=my users,dc=example,dc=org" > /dev/null 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/02-create-ldap-user.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: create-ldap-user 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ./create_ldap_user.sh openldap-0:/tmp 8 | - script: kubectl exec -n $NAMESPACE openldap-0 -- sh /tmp/create_ldap_user.sh 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-druid 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/03-install-test-druid.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-druid 6 | labels: 7 | app: test-druid 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-druid 13 | template: 14 | metadata: 15 | labels: 16 | app: test-druid 17 | spec: 18 | containers: 19 | - name: test-druid 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | command: ["sleep", "infinity"] 22 | terminationGracePeriodSeconds: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/10-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: zk 29 | --- 30 | apiVersion: zookeeper.stackable.tech/v1alpha1 31 | kind: ZookeeperZnode 32 | metadata: 33 | name: hdfs-znode 34 | spec: 35 | clusterRef: 36 | name: zk 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/11-create-authentication-classes.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: create-ldap-user 6 | commands: 7 | # We need to replace $NAMESPACE (by KUTTL) in the create-authentication-classes.yaml(.j2) 8 | - script: envsubst < 11_authentication-classes.yaml | kubectl apply -n $NAMESPACE -f - 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/11_authentication-classes.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: authentication.stackable.tech/v1alpha1 3 | kind: AuthenticationClass 4 | metadata: 5 | name: druid-ldap-auth-class-$NAMESPACE 6 | spec: 7 | provider: 8 | ldap: 9 | hostname: openldap.$NAMESPACE.svc.cluster.local 10 | searchBase: ou=my users,dc=example,dc=org 11 | searchFilter: (uid=%s) 12 | {% if test_scenario['values']['ldap-no-bind-credentials'] == 'true' %} 13 | bindCredentials: null 14 | {% else %} 15 | bindCredentials: 16 | secretClass: druid-with-ldap-bind 17 | {% endif %} 18 | {% if test_scenario['values']['ldap-use-tls'] == 'false' %} 19 | port: 1389 20 | {% else %} 21 | port: 1636 22 | tls: 23 | verification: 24 | server: 25 | caCert: 26 | secretClass: openldap-tls 27 | {% endif %} 28 | --- 29 | apiVersion: secrets.stackable.tech/v1alpha1 30 | kind: SecretClass 31 | metadata: 32 | name: druid-with-ldap-bind 33 | spec: 34 | backend: 35 | k8sSearch: 36 | searchNamespace: 37 | pod: {} 38 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/12-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-hdfs-namenode-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-hdfs-journalnode-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-hdfs-datanode-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/12-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: druid-hdfs 6 | timeout: 600 7 | --- 8 | apiVersion: hdfs.stackable.tech/v1alpha1 9 | kind: HdfsCluster 10 | metadata: 11 | name: druid-hdfs 12 | spec: 13 | image: 14 | productVersion: "{{ test_scenario['values']['hadoop-latest'] }}" 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | dfsReplication: 1 18 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 19 | vectorAggregatorConfigMapName: vector-aggregator-discovery 20 | {% endif %} 21 | zookeeperConfigMapName: hdfs-znode 22 | nameNodes: 23 | config: 24 | logging: 25 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 26 | roleGroups: 27 | default: 28 | replicas: 2 29 | dataNodes: 30 | config: 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | journalNodes: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/13-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: derby-druid-broker-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: derby-druid-coordinator-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: derby-druid-historical-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | --- 30 | apiVersion: apps/v1 31 | kind: StatefulSet 32 | metadata: 33 | name: derby-druid-middlemanager-default 34 | status: 35 | readyReplicas: 1 36 | replicas: 1 37 | --- 38 | apiVersion: apps/v1 39 | kind: StatefulSet 40 | metadata: 41 | name: derby-druid-router-default 42 | status: 43 | readyReplicas: 1 44 | replicas: 1 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE test-druid-0 -- python /tmp/authcheck.py 6 | timeout: 60 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/20-authcheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 60 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ./authcheck.py test-druid-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/create_ldap_user.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # To check the existing users 4 | # ldapsearch -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin -b "ou=my users,dc=example,dc=org" 5 | 6 | # To check the new user 7 | # ldapsearch -H ldap://localhost:1389 -D "cn=integrationtest,ou=my users,dc=example,dc=org" -w 'bindPasswordWithSpecialCharacter\@<&>"'"'" -b "ou=my users,dc=example,dc=org" 8 | 9 | cat << 'EOF' | ldapadd -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin 10 | dn: ou=my users,dc=example,dc=org 11 | ou: my users 12 | objectclass: top 13 | objectclass: organizationalUnit 14 | EOF 15 | 16 | cat << 'EOF' | ldapadd -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin 17 | dn: cn=integrationtest,ou=my users,dc=example,dc=org 18 | objectClass: inetOrgPerson 19 | objectClass: posixAccount 20 | objectClass: shadowAccount 21 | cn: integrationtest 22 | uid: integrationtest 23 | givenName: Stackable 24 | sn: Integration-Test 25 | mail: integrationtest@stackable.de 26 | uidNumber: 16842 27 | gidNumber: 100 28 | homeDirectory: /home/integrationtest 29 | loginShell: /bin/bash 30 | userPassword: {crypt}x 31 | shadowLastChange: 0 32 | shadowMax: 0 33 | shadowWarning: 0 34 | EOF 35 | 36 | ldappasswd -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin -s 'bindPasswordWithSpecialCharacter\@<&>"'"'" "cn=integrationtest,ou=my users,dc=example,dc=org" 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-vector-aggregator 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/01-install-druid-vector-aggregator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install druid-vector-aggregator vector 7 | --namespace $NAMESPACE 8 | --version 0.43.0 9 | --repo https://helm.vector.dev 10 | --values 01_druid-vector-aggregator-values.yaml 11 | --- 12 | apiVersion: v1 13 | kind: ConfigMap 14 | metadata: 15 | name: druid-vector-aggregator-discovery 16 | data: 17 | ADDRESS: druid-vector-aggregator:6123 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: hdfs-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/02-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: druid-zk 29 | --- 30 | apiVersion: zookeeper.stackable.tech/v1alpha1 31 | kind: ZookeeperZnode 32 | metadata: 33 | name: hdfs-znode 34 | spec: 35 | clusterRef: 36 | name: druid-zk 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-hdfs-namenode-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-hdfs-journalnode-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-hdfs-datanode-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/03-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: druid-hdfs 6 | timeout: 600 7 | --- 8 | apiVersion: hdfs.stackable.tech/v1alpha1 9 | kind: HdfsCluster 10 | metadata: 11 | name: druid-hdfs 12 | spec: 13 | image: 14 | productVersion: "{{ test_scenario['values']['hadoop'] }}" 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | dfsReplication: 1 18 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 19 | vectorAggregatorConfigMapName: vector-aggregator-discovery 20 | {% endif %} 21 | zookeeperConfigMapName: hdfs-znode 22 | nameNodes: 23 | config: 24 | logging: 25 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 26 | roleGroups: 27 | default: 28 | replicas: 2 29 | dataNodes: 30 | config: 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | journalNodes: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/04-create-configmap-with-prepared-logs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: > 6 | kubectl create configmap prepared-logs 7 | --from-file=prepared-logs.log4j2.xml 8 | --namespace=$NAMESPACE 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/06-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-test-runner 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/06-install-druid-test-runner.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: druid-test-runner 6 | labels: 7 | app: druid-test-runner 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: druid-test-runner 13 | template: 14 | metadata: 15 | labels: 16 | app: druid-test-runner 17 | spec: 18 | containers: 19 | - name: druid-test-runner 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | stdin: true 22 | tty: true 23 | terminationGracePeriodSeconds: 1 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/07-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: >- 6 | kubectl exec --namespace=$NAMESPACE druid-test-runner-0 -- 7 | python /tmp/test_log_aggregation.py -n $NAMESPACE 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/07-test-log-aggregation.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: | 6 | kubectl cp ./test_log_aggregation.py $NAMESPACE/druid-test-runner-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/test_log_aggregation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import requests 3 | 4 | 5 | def check_sent_events(): 6 | response = requests.post( 7 | "http://druid-vector-aggregator:8686/graphql", 8 | json={ 9 | "query": """ 10 | { 11 | transforms(first:100) { 12 | nodes { 13 | componentId 14 | metrics { 15 | sentEventsTotal { 16 | sentEventsTotal 17 | } 18 | } 19 | } 20 | } 21 | } 22 | """ 23 | }, 24 | ) 25 | 26 | assert ( 27 | response.status_code == 200 28 | ), "Cannot access the API of the vector aggregator." 29 | 30 | result = response.json() 31 | 32 | transforms = result["data"]["transforms"]["nodes"] 33 | for transform in transforms: 34 | sentEvents = transform["metrics"]["sentEventsTotal"] 35 | componentId = transform["componentId"] 36 | 37 | if componentId == "filteredInvalidEvents": 38 | assert ( 39 | sentEvents is None or sentEvents["sentEventsTotal"] == 0 40 | ), "Invalid log events were sent." 41 | else: 42 | assert ( 43 | sentEvents is not None and sentEvents["sentEventsTotal"] > 0 44 | ), f'No events were sent in "{componentId}".' 45 | 46 | 47 | if __name__ == "__main__": 48 | check_sent_events() 49 | print("Test successful!") 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/00-druid-secrets.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: druid-oidc-client 6 | timeout: 240 7 | stringData: 8 | clientId: druid 9 | clientSecret: R1bxHUD569vHeQdw 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/04-install-minio.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install minio-druid 7 | --namespace $NAMESPACE 8 | --version 12.6.4 9 | -f 04_helm-bitnami-minio-values.yaml 10 | --repo https://charts.bitnami.com/bitnami minio 11 | timeout: 240 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/04_helm-bitnami-minio-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | mode: standalone 3 | disableWebUI: false 4 | extraEnvVars: 5 | - name: BITNAMI_DEBUG 6 | value: "true" 7 | - name: MINIO_LOG_LEVEL 8 | value: DEBUG 9 | 10 | provisioning: 11 | enabled: true 12 | buckets: 13 | - name: druid 14 | usersExistingSecrets: 15 | - centralized-minio-users 16 | resources: 17 | requests: 18 | memory: 1Gi 19 | cpu: "512m" 20 | limits: 21 | memory: "1Gi" 22 | cpu: "1" 23 | podSecurityContext: 24 | enabled: false 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | volumePermissions: 29 | enabled: false 30 | 31 | podSecurityContext: 32 | enabled: false 33 | 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | persistence: 38 | enabled: false 39 | 40 | resources: 41 | requests: 42 | memory: 1Gi 43 | cpu: "512m" 44 | limits: 45 | memory: "1Gi" 46 | cpu: "1" 47 | 48 | auth: 49 | existingSecret: minio-credentials 50 | 51 | service: 52 | type: NodePort 53 | 54 | {% if test_scenario['values']['s3-use-tls'] == 'true' %} 55 | tls: 56 | enabled: true 57 | existingSecret: minio-tls-certificates 58 | {% endif %} 59 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/10-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: druid-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/20-zookeeper.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: zk 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-keycloak 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: keycloak 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/30-install-keycloak.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: | 6 | INSTANCE_NAME=keycloak \ 7 | REALM=test \ 8 | USERNAME=jane.doe \ 9 | FIRST_NAME=Jane \ 10 | LAST_NAME=Doe \ 11 | EMAIL=jane.doe@stackable.tech \ 12 | PASSWORD=T8mn72D9 \ 13 | CLIENT_ID=druid \ 14 | CLIENT_SECRET=R1bxHUD569vHeQdw \ 15 | envsubst < 30_keycloak.yaml | kubectl apply -n $NAMESPACE -f - 16 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/35-install-authentication-class.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: envsubst < 35_authentication-class.yaml | kubectl apply -n $NAMESPACE -f - 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/35_authentication-class.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: authentication.stackable.tech/v1alpha1 3 | kind: AuthenticationClass 4 | metadata: 5 | name: druid-oidc-auth-class-$NAMESPACE 6 | spec: 7 | provider: 8 | oidc: 9 | hostname: keycloak.$NAMESPACE.svc.cluster.local 10 | rootPath: /realms/test/ 11 | principalClaim: preferred_username 12 | scopes: 13 | - openid 14 | - email 15 | - profile 16 | providerHint: Keycloak 17 | {% if test_scenario['values']['oidc-use-tls'] == 'true' %} 18 | port: 8443 19 | tls: 20 | verification: 21 | server: 22 | caCert: 23 | secretClass: tls 24 | {% else %} 25 | port: 8080 26 | tls: null 27 | {% endif %} 28 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-broker-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-coordinator-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-historical-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | --- 30 | apiVersion: apps/v1 31 | kind: StatefulSet 32 | metadata: 33 | name: druid-middlemanager-default 34 | status: 35 | readyReplicas: 1 36 | replicas: 1 37 | --- 38 | apiVersion: apps/v1 39 | kind: StatefulSet 40 | metadata: 41 | name: druid-router-default 42 | status: 43 | readyReplicas: 1 44 | replicas: 1 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/40-install-druid.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: envsubst < 40_druid.yaml | kubectl apply -n $NAMESPACE -f - 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/50-create-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: kuttl.dev/v1beta1 2 | kind: TestStep 3 | commands: 4 | - script: kubectl create cm ingestion-check -n $NAMESPACE --from-file=../../../../templates/kuttl/commons/ingestioncheck-tls.py --from-file=../../../../templates/kuttl/commons/druid-quickstartimport.json 5 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/51-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-test-container 6 | timeout: 300 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: login 6 | timeout: 300 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE python-0 -- python /stackable/login.py $NAMESPACE 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/60-login.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: > 6 | envsubst '$NAMESPACE' < login.py | 7 | kubectl exec -n $NAMESPACE -i python-0 -- tee /stackable/login.py > /dev/null 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/70-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: ingestion-check 6 | timeout: 300 7 | commands: 8 | - script: kubectl exec python-0 -n $NAMESPACE -- python /tmp/ingestioncheck-tls.py $NAMESPACE druid internal-and-server-tls true 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/login.py: -------------------------------------------------------------------------------- 1 | # $NAMESPACE will be replaced with the namespace of the test case. 2 | 3 | import logging 4 | import os 5 | import requests 6 | import sys 7 | from bs4 import BeautifulSoup 8 | 9 | logging.basicConfig( 10 | level="DEBUG", format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stdout 11 | ) 12 | 13 | namespace = sys.argv[1] 14 | tls = os.environ["OIDC_USE_TLS"] 15 | 16 | session = requests.Session() 17 | 18 | druid_router_service = f"druid-router-default.{namespace}.svc.cluster.local" 19 | keycloak_service = f"keycloak.{namespace}.svc.cluster.local" 20 | 21 | # Open Druid web UI which will redirect to OIDC login 22 | login_page = session.get( 23 | f"https://{druid_router_service}:9088/unified-console.html", 24 | verify=False, 25 | headers={"Content-type": "application/json"}, 26 | ) 27 | keycloak_base_url = ( 28 | f"https://{keycloak_service}:8443" 29 | if tls == "true" 30 | else f"http://{keycloak_service}:8080" 31 | ) 32 | assert login_page.ok, "Redirection from Druid to Keycloak failed" 33 | assert login_page.url.startswith( 34 | f"{keycloak_base_url}/realms/test/protocol/openid-connect/auth?scope=openid+email+profile&response_type=code&redirect_uri=https%3A%2F%2F{druid_router_service}%3A9088%2Fdruid-ext%2Fdruid-pac4j%2Fcallback&state=" 35 | ), "Redirection to Keycloak expected" 36 | 37 | # Login to keycloak with test user 38 | login_page_html = BeautifulSoup(login_page.text, "html.parser") 39 | authenticate_url = login_page_html.form["action"] 40 | welcome_page = session.post( 41 | authenticate_url, data={"username": "jane.doe", "password": "T8mn72D9"} 42 | ) 43 | 44 | assert welcome_page.ok, "Login failed" 45 | assert ( 46 | welcome_page.url == f"https://{druid_router_service}:9088/unified-console.html" 47 | ), "Redirection to the Druid web UI expected" 48 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: hdfs-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/01-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: druid-zk 29 | --- 30 | apiVersion: zookeeper.stackable.tech/v1alpha1 31 | kind: ZookeeperZnode 32 | metadata: 33 | name: hdfs-znode 34 | spec: 35 | clusterRef: 36 | name: druid-zk 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-hdfs-namenode-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-hdfs-journalnode-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-hdfs-datanode-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/02-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: druid-hdfs 6 | timeout: 600 7 | --- 8 | apiVersion: hdfs.stackable.tech/v1alpha1 9 | kind: HdfsCluster 10 | metadata: 11 | name: druid-hdfs 12 | spec: 13 | image: 14 | productVersion: "{{ test_scenario['values']['hadoop'] }}" 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | dfsReplication: 1 18 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 19 | vectorAggregatorConfigMapName: vector-aggregator-discovery 20 | {% endif %} 21 | zookeeperConfigMapName: hdfs-znode 22 | nameNodes: 23 | config: 24 | logging: 25 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 26 | roleGroups: 27 | default: 28 | replicas: 2 29 | dataNodes: 30 | config: 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | journalNodes: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: derby-druid-broker-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: derby-druid-coordinator-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: derby-druid-historical-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | --- 30 | apiVersion: apps/v1 31 | kind: StatefulSet 32 | metadata: 33 | name: derby-druid-middlemanager-default 34 | status: 35 | readyReplicas: 1 36 | replicas: 1 37 | --- 38 | apiVersion: apps/v1 39 | kind: StatefulSet 40 | metadata: 41 | name: derby-druid-router-default 42 | status: 43 | readyReplicas: 1 44 | replicas: 1 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/04-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 180 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: derby-druid-broker-newrg 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: derby-druid-coordinator-newrg 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: derby-druid-historical-newrg 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | --- 30 | apiVersion: apps/v1 31 | kind: StatefulSet 32 | metadata: 33 | name: derby-druid-middlemanager-newrg 34 | status: 35 | readyReplicas: 1 36 | replicas: 1 37 | --- 38 | apiVersion: apps/v1 39 | kind: StatefulSet 40 | metadata: 41 | name: derby-druid-router-newrg 42 | status: 43 | readyReplicas: 1 44 | replicas: 1 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/04-errors.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: derby-druid-broker-default 6 | --- 7 | apiVersion: apps/v1 8 | kind: StatefulSet 9 | metadata: 10 | name: derby-druid-coordinator-default 11 | --- 12 | apiVersion: apps/v1 13 | kind: StatefulSet 14 | metadata: 15 | name: derby-druid-historical-default 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: derby-druid-middlemanager-default 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: derby-druid-router-default 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: druid-zk-server-default 5 | status: 6 | availableReplicas: 1 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/00-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | servers: 10 | roleGroups: 11 | default: 12 | replicas: 1 13 | --- 14 | apiVersion: zookeeper.stackable.tech/v1alpha1 15 | kind: ZookeeperZnode 16 | metadata: 17 | name: druid-znode 18 | spec: 19 | clusterRef: 20 | name: druid-zk 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: hdfs-znode 26 | spec: 27 | clusterRef: 28 | name: druid-zk 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: hdfs-namenode-default 6 | status: 7 | availableReplicas: 2 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: hdfs-datanode-default 13 | status: 14 | availableReplicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: hdfs-journalnode-default 20 | status: 21 | availableReplicas: 1 22 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/10-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: hdfs.stackable.tech/v1alpha1 3 | kind: HdfsCluster 4 | metadata: 5 | name: hdfs 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['hadoop-latest'] }}" 9 | clusterConfig: 10 | dfsReplication: 1 11 | zookeeperConfigMapName: hdfs-znode 12 | nameNodes: 13 | roleGroups: 14 | default: 15 | replicas: 2 16 | dataNodes: 17 | roleGroups: 18 | default: 19 | replicas: 1 20 | journalNodes: 21 | roleGroups: 22 | default: 23 | replicas: 1 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: druid-broker-default 6 | spec: 7 | template: 8 | metadata: 9 | labels: 10 | level: role 11 | status: 12 | readyReplicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-coordinator-default 18 | spec: 19 | template: 20 | metadata: 21 | labels: 22 | level: role-group 23 | status: 24 | readyReplicas: 1 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: druid-historical-default 30 | status: 31 | readyReplicas: 1 32 | --- 33 | apiVersion: apps/v1 34 | kind: StatefulSet 35 | metadata: 36 | name: druid-middlemanager-default 37 | status: 38 | readyReplicas: 1 39 | --- 40 | apiVersion: apps/v1 41 | kind: StatefulSet 42 | metadata: 43 | name: druid-router-default 44 | status: 45 | readyReplicas: 1 46 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | --- 5 | apiVersion: v1 6 | kind: Secret 7 | metadata: 8 | name: minio-credentials 9 | --- 10 | apiVersion: v1 11 | kind: Secret 12 | metadata: 13 | name: centralized-minio-users 14 | {% if test_scenario['values']['s3-use-tls'] == 'true' %} 15 | --- 16 | apiVersion: v1 17 | kind: Secret 18 | metadata: 19 | name: minio-tls-certificates 20 | {% endif %} 21 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 22 | --- 23 | apiVersion: v1 24 | kind: ConfigMap 25 | metadata: 26 | name: vector-aggregator-discovery 27 | {% endif %} 28 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/02-install-minio.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install minio-druid 7 | --namespace $NAMESPACE 8 | --version 12.6.4 9 | -f 02_helm-bitnami-minio-values.yaml 10 | --repo https://charts.bitnami.com/bitnami minio 11 | timeout: 240 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/02_helm-bitnami-minio-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | mode: standalone 3 | disableWebUI: false 4 | extraEnvVars: 5 | - name: BITNAMI_DEBUG 6 | value: "true" 7 | - name: MINIO_LOG_LEVEL 8 | value: DEBUG 9 | 10 | provisioning: 11 | enabled: true 12 | buckets: 13 | - name: druid 14 | usersExistingSecrets: 15 | - centralized-minio-users 16 | resources: 17 | requests: 18 | memory: 1Gi 19 | cpu: "512m" 20 | limits: 21 | memory: "1Gi" 22 | cpu: "1" 23 | podSecurityContext: 24 | enabled: false 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | volumePermissions: 29 | enabled: false 30 | 31 | podSecurityContext: 32 | enabled: false 33 | 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | persistence: 38 | enabled: false 39 | 40 | resources: 41 | requests: 42 | memory: 1Gi 43 | cpu: "512m" 44 | limits: 45 | memory: "1Gi" 46 | cpu: "1" 47 | 48 | auth: 49 | existingSecret: minio-credentials 50 | 51 | service: 52 | type: NodePort 53 | 54 | {% if test_scenario['values']['s3-use-tls'] == 'true' %} 55 | tls: 56 | enabled: true 57 | existingSecret: minio-tls-certificates 58 | {% endif %} 59 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: druid-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/10-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: druid-zk 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | commands: 6 | - script: kubectl get cm -n $NAMESPACE druid-resources-broker-default -o yaml | grep -- '-Xmx324m' | xargs test ! -z 7 | - script: kubectl get cm -n $NAMESPACE druid-resources-coordinator-default -o yaml | grep -- '-Xmx724m' | xargs test ! -z 8 | - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- '-Xmx2847m' | xargs test ! -z 9 | - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- 'druid.processing.numThreads=1' | xargs test ! -z 10 | - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- 'druid.processing.numMergeBuffers=2' | xargs test ! -z 11 | - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- 'druid.processing.buffer.sizeBytes=242944Ki' | xargs test ! -z 12 | - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-default -o yaml | grep -- '-Xmx724m' | xargs test ! -z 13 | - script: kubectl get cm -n $NAMESPACE druid-resources-router-default -o yaml | grep -- '-Xmx596m' | xargs test ! -z 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: minio-credentials 6 | --- 7 | apiVersion: v1 8 | kind: Secret 9 | metadata: 10 | name: centralized-minio-users 11 | {% if test_scenario['values']['s3-use-tls'] == 'true' %} 12 | --- 13 | apiVersion: v1 14 | kind: Secret 15 | metadata: 16 | name: minio-tls-certificates 17 | {% endif %} 18 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 19 | --- 20 | apiVersion: v1 21 | kind: ConfigMap 22 | metadata: 23 | name: vector-aggregator-discovery 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: druid-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/01-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | config: 22 | myidOffset: 10 23 | --- 24 | apiVersion: zookeeper.stackable.tech/v1alpha1 25 | kind: ZookeeperZnode 26 | metadata: 27 | name: druid-znode 28 | spec: 29 | clusterRef: 30 | name: druid-zk 31 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/04-install-minio.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install minio-druid 7 | --namespace $NAMESPACE 8 | --version 12.6.4 9 | -f 04_helm-bitnami-minio-values.yaml 10 | --repo https://charts.bitnami.com/bitnami minio 11 | timeout: 240 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/04_helm-bitnami-minio-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | mode: standalone 3 | disableWebUI: false 4 | extraEnvVars: 5 | - name: BITNAMI_DEBUG 6 | value: "true" 7 | - name: MINIO_LOG_LEVEL 8 | value: DEBUG 9 | 10 | provisioning: 11 | enabled: true 12 | buckets: 13 | - name: druid 14 | usersExistingSecrets: 15 | - centralized-minio-users 16 | resources: 17 | requests: 18 | memory: 1Gi 19 | cpu: "512m" 20 | limits: 21 | memory: "1Gi" 22 | cpu: "1" 23 | podSecurityContext: 24 | enabled: false 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | volumePermissions: 29 | enabled: false 30 | 31 | podSecurityContext: 32 | enabled: false 33 | 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | persistence: 38 | enabled: false 39 | 40 | resources: 41 | requests: 42 | memory: 1Gi 43 | cpu: "512m" 44 | limits: 45 | memory: "1Gi" 46 | cpu: "1" 47 | 48 | auth: 49 | existingSecret: minio-credentials 50 | 51 | service: 52 | type: NodePort 53 | 54 | {% if test_scenario['values']['s3-use-tls'] == 'true' %} 55 | tls: 56 | enabled: true 57 | existingSecret: minio-tls-certificates 58 | {% endif %} 59 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/07-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: checks 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/07-checks-container.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: checks 6 | labels: 7 | app: checks 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: checks 13 | template: 14 | metadata: 15 | labels: 16 | app: checks 17 | spec: 18 | containers: 19 | - name: checks 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | command: ["sleep", "infinity"] 22 | terminationGracePeriodSeconds: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-s3-deep-storage-broker-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-s3-deep-storage-coordinator-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-s3-deep-storage-historical-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | --- 30 | apiVersion: apps/v1 31 | kind: StatefulSet 32 | metadata: 33 | name: druid-s3-deep-storage-middlemanager-default 34 | status: 35 | readyReplicas: 1 36 | replicas: 1 37 | --- 38 | apiVersion: apps/v1 39 | kind: StatefulSet 40 | metadata: 41 | name: druid-s3-deep-storage-router-default 42 | status: 43 | readyReplicas: 1 44 | replicas: 1 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/11-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/healthcheck.py druid-s3-deep-storage 6 | timeout: 300 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/11-healthcheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/healthcheck.py checks-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/12-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/ingestioncheck.py druid-s3-deep-storage 6 | timeout: 300 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/12-ingestioncheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/ingestioncheck.py checks-0:/tmp 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/druid-quickstartimport.json checks-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/certs/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID2TCCAsGgAwIBAgIUK0v4Bnm2z6agvriRjolTTuYdlVEwDQYJKoZIhvcNAQEL 3 | BQAwezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1Ib2xzdGVpbjEO 4 | MAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJsZSBTaWduaW5nIEF1dGhv 5 | cml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTAgFw0yMzA2MjEwNjU5MDFa 6 | GA8yMTIzMDUyODA2NTkwMVowezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxl 7 | c3dpZy1Ib2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJs 8 | ZSBTaWduaW5nIEF1dGhvcml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTCC 9 | ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALKoAYBaLkcctsjbfKnq84VD 10 | VvHmme7sDj0fPE7cn4APDMbg2XDmIbu48i4V5UkcnRSu9Lvg5mCvfGVhXsXXPTIm 11 | lpLLHl9KRU/Tx3YNfCqVA0rsiDVP0l6XEa6TadGb0YEX/F1XeYcN6YJeE1XvD/l4 12 | lqzJ22ASyz0bTmIPrLIuXJ52lu1ernKnuM3fh+U6x/TmCdExhrxgxcGnvnacQEzZ 13 | vHyCFAe1lNhVSBqoDy3nbQ4OOppwdQTLe+h4DwrXvIr7uZ9LfO5kYuUrGQsGQPkf 14 | uN1YTmjdAhzrXhXLfn5G5tGEEL6dJusuWX0zgUbmUxND5TtU5mxg5FloKqSkAUkC 15 | AwEAAaNTMFEwHQYDVR0OBBYEFOsu4qvrVxHjQ8uvk+n2lQfQA4hYMB8GA1UdIwQY 16 | MBaAFOsu4qvrVxHjQ8uvk+n2lQfQA4hYMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI 17 | hvcNAQELBQADggEBABX/5Ha76DfFdQeDR7y96cOaB1UYbLioIFbx0syVdOVW92Ji 18 | aNicofWfjcKzvJ2s6Vzb0i8iHneebAGkGI8SG4ek4rfLbb6ESijSoqpsfaMkryqS 19 | L4rMhMQy9pFcrO7eMDcKHDksyNJPnw/jRyYKh5/MZ+i+E35/AkGwPVMTEVHOdjFZ 20 | CtlHEUN6S825j5UatHXFewe1hGKZhED3cl8vVfCNlVdMEOfxu6H0NOLLU9b95LJE 21 | rAfLmggdMa0rhvojqj/6yyAGNJUfsRQEHOMDxjAJsE4GFBOrH1mK3dToeumYzYkP 22 | tayxFD2r4zHUvyM8tFMURX4otX/TEM1HILdhvic= 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/certs/client.crt.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID1DCCArygAwIBAgIUCTFIFj8CVs1zZUTecHq+u4/PhpQwDQYJKoZIhvcNAQEL 3 | BQAwezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1Ib2xzdGVpbjEO 4 | MAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJsZSBTaWduaW5nIEF1dGhv 5 | cml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTAgFw0yMzA2MjEwNjU5MDJa 6 | GA8yMTIzMDUyODA2NTkwMlowZDELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxl 7 | c3dpZy1Ib2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxEjAQBgNVBAoMCVN0YWNrYWJs 8 | ZTEUMBIGA1UEAwwLbWluaW8tZHJ1aWQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw 9 | ggEKAoIBAQCymEOa0tUceMQnsya8Fyu3FrUrx8qM0TAv6WSf/Obt5jlDJ4IHOkrh 10 | uy1HvSGhRrtH04MlLe9rMgOLIIC+l0eiFbZ+Y0J6omusYKsbAaS69xIF9sMI0byL 11 | wSVCoajoGWSd3sarLIEqK94rzc25pmBMWzl9211LrQrmRmdqm880Wq8EWVBq7dAD 12 | BvUWQJPeIZbvzdc//4rXet+kKhJL6cQubcV8J0kZH9qOmDWfpWwfmHcHPK1ipbsM 13 | 2I07RwHj7ipc1Uxy32hKWt2D6viFjQlldvwUC7ZiQzsq47iYtJkoLOOwUtqc6wTi 14 | UPbnpCt4SfKgtUBtS2BgN/HYylVOE5ztAgMBAAGjZTBjMCEGA1UdEQQaMBiCC21p 15 | bmlvLWRydWlkgglsb2NhbGhvc3QwHQYDVR0OBBYEFFw2l1RDtz7KKdyOI6/tdHKc 16 | hElMMB8GA1UdIwQYMBaAFOsu4qvrVxHjQ8uvk+n2lQfQA4hYMA0GCSqGSIb3DQEB 17 | CwUAA4IBAQBe4VrH/hM1pfKiQIsXoWQPq9jBcHuIp35wYrjEcOvIQEFJIBg87l71 18 | UXwHeKZYf3dU0e3xoDFFkJ7Fqq4o5IMBfnRcJF+AdTavjpmacFlo/hWTF99qko+Q 19 | GTZf0G5NoS6R7n7jTPUXdlmI+BvT60SXDIt1tYdaZgspoLB6NSj5dcDeJ+/j6r98 20 | LRqKr39LNHr7Xx7TBuOFGUo8UO9O+EZ5j1l8G/pdODeF1pCSEgKMEWgkZIfCg0io 21 | iPkhyMO1NoEgTSPLrdG9Y/2SlEu9+s6B6WNBn3JL5QsWpZsuObwatInxUn+xm8WF 22 | 07voG4u9yPfFVhp+GoJkqhjexacLSIGc 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/certs/client.csr.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIIC3TCCAcUCAQAwZDELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1I 3 | b2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxEjAQBgNVBAoMCVN0YWNrYWJsZTEUMBIG 4 | A1UEAwwLbWluaW8tZHJ1aWQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB 5 | AQCymEOa0tUceMQnsya8Fyu3FrUrx8qM0TAv6WSf/Obt5jlDJ4IHOkrhuy1HvSGh 6 | RrtH04MlLe9rMgOLIIC+l0eiFbZ+Y0J6omusYKsbAaS69xIF9sMI0byLwSVCoajo 7 | GWSd3sarLIEqK94rzc25pmBMWzl9211LrQrmRmdqm880Wq8EWVBq7dADBvUWQJPe 8 | IZbvzdc//4rXet+kKhJL6cQubcV8J0kZH9qOmDWfpWwfmHcHPK1ipbsM2I07RwHj 9 | 7ipc1Uxy32hKWt2D6viFjQlldvwUC7ZiQzsq47iYtJkoLOOwUtqc6wTiUPbnpCt4 10 | SfKgtUBtS2BgN/HYylVOE5ztAgMBAAGgNDAyBgkqhkiG9w0BCQ4xJTAjMCEGA1Ud 11 | EQQaMBiCC21pbmlvLWRydWlkgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggEB 12 | AJkXuDtpKepocy4R6AfdAQjVM5rNicdGmZpVHIor3x4dn4+sD6XJ220xjX8qe3W7 13 | +oGKamfiXEe9hVMUAlmtRA8/LPfTyzpH+XUkp9+LapkwwECTedzhZINdUKtwE6eT 14 | EwPfEgHiM+NRDg8u1yXi8ZitASFPOLZ3Fba9kmLMcCbvWHEoVtZKg2GUw+ERuAPY 15 | WIoBOqPjjYfgO4IAxdZuSbO1//tQtrvV8OQ9Yj9cP7Vl09hGZjne3YUGpPiD6RnT 16 | EB84lyC6fFs21JrqUOcG6KY2Xool7VQdHD8ml4cs4EMrP8P9CqOKAx/hDBd3uFK5 17 | a9zLwpjgFTIlnnCrCQRgE/I= 18 | -----END CERTIFICATE REQUEST----- 19 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/certs/generate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Creating Root Certificate Authority" 4 | openssl genrsa \ 5 | -out root-ca.key.pem \ 6 | 2048 7 | 8 | echo "Self-signing the Root Certificate Authority" 9 | openssl req \ 10 | -x509 \ 11 | -new \ 12 | -nodes \ 13 | -key root-ca.key.pem \ 14 | -days 36500 \ 15 | -out root-ca.crt.pem \ 16 | -subj "/C=DE/ST=Schleswig-Holstein/L=Wedel/O=Stackable Signing Authority Inc/CN=stackable.de" 17 | 18 | echo "Creating client cert" 19 | FQDN="minio-druid" 20 | 21 | openssl genrsa \ 22 | -out client.key.pem \ 23 | 2048 24 | 25 | echo "Creating the CSR" 26 | openssl req -new \ 27 | -key client.key.pem \ 28 | -out client.csr.pem \ 29 | -subj "/C=DE/ST=Schleswig-Holstein/L=Wedel/O=Stackable/CN=${FQDN}" \ 30 | -addext "subjectAltName = DNS:${FQDN}, DNS:localhost" 31 | 32 | echo "Signing the client cert with the root ca" 33 | openssl x509 \ 34 | -req -in client.csr.pem \ 35 | -CA root-ca.crt.pem \ 36 | -CAkey root-ca.key.pem \ 37 | -CAcreateserial \ 38 | -out client.crt.pem \ 39 | -days 36500 \ 40 | -copy_extensions copy 41 | 42 | 43 | echo "Copying the files to match the api of the secret-operator" 44 | cp root-ca.crt.pem ca.crt 45 | cp client.key.pem tls.key 46 | cp client.crt.pem tls.crt 47 | 48 | echo "To create a k8s secret run" 49 | echo "kubectl create secret generic foo --from-file=ca.crt --from-file=tls.crt --from-file=tls.key" 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/certs/root-ca.crt.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID2TCCAsGgAwIBAgIUK0v4Bnm2z6agvriRjolTTuYdlVEwDQYJKoZIhvcNAQEL 3 | BQAwezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1Ib2xzdGVpbjEO 4 | MAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJsZSBTaWduaW5nIEF1dGhv 5 | cml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTAgFw0yMzA2MjEwNjU5MDFa 6 | GA8yMTIzMDUyODA2NTkwMVowezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxl 7 | c3dpZy1Ib2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJs 8 | ZSBTaWduaW5nIEF1dGhvcml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTCC 9 | ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALKoAYBaLkcctsjbfKnq84VD 10 | VvHmme7sDj0fPE7cn4APDMbg2XDmIbu48i4V5UkcnRSu9Lvg5mCvfGVhXsXXPTIm 11 | lpLLHl9KRU/Tx3YNfCqVA0rsiDVP0l6XEa6TadGb0YEX/F1XeYcN6YJeE1XvD/l4 12 | lqzJ22ASyz0bTmIPrLIuXJ52lu1ernKnuM3fh+U6x/TmCdExhrxgxcGnvnacQEzZ 13 | vHyCFAe1lNhVSBqoDy3nbQ4OOppwdQTLe+h4DwrXvIr7uZ9LfO5kYuUrGQsGQPkf 14 | uN1YTmjdAhzrXhXLfn5G5tGEEL6dJusuWX0zgUbmUxND5TtU5mxg5FloKqSkAUkC 15 | AwEAAaNTMFEwHQYDVR0OBBYEFOsu4qvrVxHjQ8uvk+n2lQfQA4hYMB8GA1UdIwQY 16 | MBaAFOsu4qvrVxHjQ8uvk+n2lQfQA4hYMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI 17 | hvcNAQELBQADggEBABX/5Ha76DfFdQeDR7y96cOaB1UYbLioIFbx0syVdOVW92Ji 18 | aNicofWfjcKzvJ2s6Vzb0i8iHneebAGkGI8SG4ek4rfLbb6ESijSoqpsfaMkryqS 19 | L4rMhMQy9pFcrO7eMDcKHDksyNJPnw/jRyYKh5/MZ+i+E35/AkGwPVMTEVHOdjFZ 20 | CtlHEUN6S825j5UatHXFewe1hGKZhED3cl8vVfCNlVdMEOfxu6H0NOLLU9b95LJE 21 | rAfLmggdMa0rhvojqj/6yyAGNJUfsRQEHOMDxjAJsE4GFBOrH1mK3dToeumYzYkP 22 | tayxFD2r4zHUvyM8tFMURX4otX/TEM1HILdhvic= 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/certs/root-ca.crt.srl: -------------------------------------------------------------------------------- 1 | 72A64560889B6BDE4797487BFBD33AE9F234146E 2 | -------------------------------------------------------------------------------- /tests/templates/kuttl/s3-deep-storage/certs/tls.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID1DCCArygAwIBAgIUCTFIFj8CVs1zZUTecHq+u4/PhpQwDQYJKoZIhvcNAQEL 3 | BQAwezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1Ib2xzdGVpbjEO 4 | MAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJsZSBTaWduaW5nIEF1dGhv 5 | cml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTAgFw0yMzA2MjEwNjU5MDJa 6 | GA8yMTIzMDUyODA2NTkwMlowZDELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxl 7 | c3dpZy1Ib2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxEjAQBgNVBAoMCVN0YWNrYWJs 8 | ZTEUMBIGA1UEAwwLbWluaW8tZHJ1aWQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw 9 | ggEKAoIBAQCymEOa0tUceMQnsya8Fyu3FrUrx8qM0TAv6WSf/Obt5jlDJ4IHOkrh 10 | uy1HvSGhRrtH04MlLe9rMgOLIIC+l0eiFbZ+Y0J6omusYKsbAaS69xIF9sMI0byL 11 | wSVCoajoGWSd3sarLIEqK94rzc25pmBMWzl9211LrQrmRmdqm880Wq8EWVBq7dAD 12 | BvUWQJPeIZbvzdc//4rXet+kKhJL6cQubcV8J0kZH9qOmDWfpWwfmHcHPK1ipbsM 13 | 2I07RwHj7ipc1Uxy32hKWt2D6viFjQlldvwUC7ZiQzsq47iYtJkoLOOwUtqc6wTi 14 | UPbnpCt4SfKgtUBtS2BgN/HYylVOE5ztAgMBAAGjZTBjMCEGA1UdEQQaMBiCC21p 15 | bmlvLWRydWlkgglsb2NhbGhvc3QwHQYDVR0OBBYEFFw2l1RDtz7KKdyOI6/tdHKc 16 | hElMMB8GA1UdIwQYMBaAFOsu4qvrVxHjQ8uvk+n2lQfQA4hYMA0GCSqGSIb3DQEB 17 | CwUAA4IBAQBe4VrH/hM1pfKiQIsXoWQPq9jBcHuIp35wYrjEcOvIQEFJIBg87l71 18 | UXwHeKZYf3dU0e3xoDFFkJ7Fqq4o5IMBfnRcJF+AdTavjpmacFlo/hWTF99qko+Q 19 | GTZf0G5NoS6R7n7jTPUXdlmI+BvT60SXDIt1tYdaZgspoLB6NSj5dcDeJ+/j6r98 20 | LRqKr39LNHr7Xx7TBuOFGUo8UO9O+EZ5j1l8G/pdODeF1pCSEgKMEWgkZIfCg0io 21 | iPkhyMO1NoEgTSPLrdG9Y/2SlEu9+s6B6WNBn3JL5QsWpZsuObwatInxUn+xm8WF 22 | 07voG4u9yPfFVhp+GoJkqhjexacLSIGc 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/00-limit-range.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: LimitRange 4 | metadata: 5 | name: limit-request-ratio 6 | spec: 7 | limits: 8 | - type: "Container" 9 | maxLimitRequestRatio: 10 | cpu: 5 11 | memory: 1 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/10-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 480 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-postgresql 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/20-install-postgres.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install druid-postgresql 7 | --namespace $NAMESPACE 8 | --version 12.5.6 9 | -f 20_helm-bitnami-postgresql-values.yaml 10 | --repo https://charts.bitnami.com/bitnami postgresql 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/20_helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | extendedConfiguration: | 9 | password_encryption=md5 10 | podSecurityContext: 11 | {% if test_scenario['values']['openshift'] == 'true' %} 12 | enabled: false 13 | {% else %} 14 | enabled: true 15 | {% endif %} 16 | containerSecurityContext: 17 | enabled: false 18 | resources: 19 | requests: 20 | memory: "128Mi" 21 | cpu: "512m" 22 | limits: 23 | memory: "128Mi" 24 | cpu: "1" 25 | auth: 26 | username: druid 27 | password: druid 28 | database: druid 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: hdfs-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/30-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: druid-znode 26 | spec: 27 | clusterRef: 28 | name: druid-zk 29 | --- 30 | apiVersion: zookeeper.stackable.tech/v1alpha1 31 | kind: ZookeeperZnode 32 | metadata: 33 | name: hdfs-znode 34 | spec: 35 | clusterRef: 36 | name: druid-zk 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-hdfs-namenode-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: druid-hdfs-journalnode-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: druid-hdfs-datanode-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/40-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: druid-hdfs 6 | timeout: 600 7 | --- 8 | apiVersion: hdfs.stackable.tech/v1alpha1 9 | kind: HdfsCluster 10 | metadata: 11 | name: druid-hdfs 12 | spec: 13 | image: 14 | productVersion: "{{ test_scenario['values']['hadoop'] }}" 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | dfsReplication: 1 18 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 19 | vectorAggregatorConfigMapName: vector-aggregator-discovery 20 | {% endif %} 21 | zookeeperConfigMapName: hdfs-znode 22 | nameNodes: 23 | config: 24 | logging: 25 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 26 | roleGroups: 27 | default: 28 | replicas: 2 29 | dataNodes: 30 | config: 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | journalNodes: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/51-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # This test checks if the containerdebug-state.json file is present and valid 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | timeout: 600 6 | commands: 7 | - script: kubectl exec -n $NAMESPACE --container druid druid-coordinator-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' 8 | - script: kubectl exec -n $NAMESPACE --container druid druid-router-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' 9 | - script: kubectl exec -n $NAMESPACE --container druid druid-middlemanager-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' 10 | - script: kubectl exec -n $NAMESPACE --container druid druid-router-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' 11 | - script: kubectl exec -n $NAMESPACE --container druid druid-historical-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: checks 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/60-checks-container.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: checks 6 | labels: 7 | app: checks 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: checks 13 | template: 14 | metadata: 15 | labels: 16 | app: checks 17 | spec: 18 | containers: 19 | - name: checks 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | command: ["sleep", "infinity"] 22 | resources: 23 | requests: 24 | memory: "128Mi" 25 | cpu: "512m" 26 | limits: 27 | memory: "128Mi" 28 | cpu: "1" 29 | terminationGracePeriodSeconds: 1 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/70-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/healthcheck.py druid 6 | timeout: 300 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/70-healthcheck.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/healthcheck.py checks-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: minio-credentials 6 | --- 7 | apiVersion: v1 8 | kind: Secret 9 | metadata: 10 | name: centralized-minio-users 11 | {% if test_scenario['values']['s3-use-tls'] == 'true' %} 12 | --- 13 | apiVersion: v1 14 | kind: Secret 15 | metadata: 16 | name: minio-tls-certificates 17 | {% endif %} 18 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 19 | --- 20 | apiVersion: v1 21 | kind: ConfigMap 22 | metadata: 23 | name: vector-aggregator-discovery 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: druid-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/01-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: druid-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 10 | clusterConfig: 11 | vectorAggregatorConfigMapName: vector-aggregator-discovery 12 | {% endif %} 13 | servers: 14 | config: 15 | logging: 16 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 17 | roleGroups: 18 | default: 19 | replicas: 1 20 | --- 21 | apiVersion: zookeeper.stackable.tech/v1alpha1 22 | kind: ZookeeperZnode 23 | metadata: 24 | name: druid-znode 25 | spec: 26 | clusterRef: 27 | name: druid-zk 28 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/03-install-minio.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install minio-druid 7 | --namespace $NAMESPACE 8 | --version 12.6.4 9 | -f 03_helm-bitnami-minio-values.yaml 10 | --repo https://charts.bitnami.com/bitnami minio 11 | timeout: 240 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/03_helm-bitnami-minio-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | mode: standalone 3 | disableWebUI: false 4 | extraEnvVars: 5 | - name: BITNAMI_DEBUG 6 | value: "true" 7 | - name: MINIO_LOG_LEVEL 8 | value: DEBUG 9 | 10 | provisioning: 11 | enabled: true 12 | buckets: 13 | - name: druid 14 | usersExistingSecrets: 15 | - centralized-minio-users 16 | resources: 17 | requests: 18 | memory: 1Gi 19 | cpu: "512m" 20 | limits: 21 | memory: "1Gi" 22 | cpu: "1" 23 | podSecurityContext: 24 | enabled: false 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | volumePermissions: 29 | enabled: false 30 | 31 | podSecurityContext: 32 | enabled: false 33 | 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | persistence: 38 | enabled: false 39 | 40 | resources: 41 | requests: 42 | memory: 1Gi 43 | cpu: "512m" 44 | limits: 45 | memory: "1Gi" 46 | cpu: "1" 47 | 48 | auth: 49 | existingSecret: minio-credentials 50 | 51 | service: 52 | type: NodePort 53 | 54 | tls: 55 | enabled: true 56 | existingSecret: minio-tls-certificates 57 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: druid-checks 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl exec -n $NAMESPACE druid-checks-0 -- /tmp/check-tls.sh $NAMESPACE {{ test_scenario['values']['tls-mode'] }} 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/10-tls-checks.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ./check-tls.sh druid-checks-0:/tmp/check-tls.sh 7 | - script: kubectl cp -n $NAMESPACE ./untrusted-ca.crt druid-checks-0:/tmp/untrusted-ca.crt 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/11-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl exec -n $NAMESPACE druid-checks-0 -- python /tmp/ingestioncheck-tls.py $NAMESPACE derby-druid {{ test_scenario['values']['tls-mode'] }} 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/11-ingestion-checks.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/ingestioncheck-tls.py druid-checks-0:/tmp 7 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/druid-quickstartimport.json druid-checks-0:/tmp 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/tls/untrusted-ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID2TCCAsGgAwIBAgIUBIxXEL+7zEuWUcO0r4nPvYFi6xAwDQYJKoZIhvcNAQEL 3 | BQAwezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1Ib2xzdGVpbjEO 4 | MAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJsZSBTaWduaW5nIEF1dGhv 5 | cml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTAgFw0yMjA2MjgxNDI1NDJa 6 | GA8yMTIyMDYwNDE0MjU0MlowezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxl 7 | c3dpZy1Ib2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJs 8 | ZSBTaWduaW5nIEF1dGhvcml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTCC 9 | ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALOBnDXGhMt8QV9TZsArVVtP 10 | wp9MxM0DFtUIFc7sbL7WcpkIWkjDZ78L6+fCUnpjeInaFefGsTMBt66daPdZ/grI 11 | 37hRnw/Fd06CcRoqROMivQEYz5xuQtalAVdqteMVPR6pS1g19J0s09ZD3LuJrICe 12 | sW4MzsyGaoz/zKSp6+8JDJKCB3qXIAFWQOCa3oOmoSe86TtN4MSuxWvKqOmUeA65 13 | vwj8DJrYq3sw1291OtCHW+Hoyiai2pp0ofaSajA1gsASa+wrXwqU8cyAOKk0N5Xs 14 | lyewdwHBCAka87FDCRMUI9FgjsDk/kzY/Hw/PKMuGFPt4hpIvX0zE+aTuPhyOrEC 15 | AwEAAaNTMFEwHQYDVR0OBBYEFNDTcQYOC8ULaK5GWVeqJllckTc8MB8GA1UdIwQY 16 | MBaAFNDTcQYOC8ULaK5GWVeqJllckTc8MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI 17 | hvcNAQELBQADggEBAHLzyAQKrbDBQNXX2smWlqX/2JAWM0xSCUGhlgCQITjdrzcv 18 | q9g0h/U6RoEEJppaFi5f4ReoqNtMa+eMvmq+Nt8Xt7c1+gJ0fQn08vok8buqiFtI 19 | BSDpwVs65D98DMThQXksGToScOIhFJU8vpUtt79CmbukGyw/uc49rBqejb4xHTPi 20 | srWPJIQkfpPmFAPkjXWsaX7rvXsGABOdfp/qMM3e2X4aO5owe8AihqmKE3XvhmeC 21 | v045+nIoFLz4mfGHiuaWK4Rpwu9HL3jHDdE4Qyn1ZEwvtQD7zE05sdUslv/zJ8gR 22 | jzo+8memTnv8W2/QfmLVnes2TKW3kVjn2YPtNRE= 23 | -----END CERTIFICATE----- 24 | --------------------------------------------------------------------------------