├── .actionlint.yaml ├── .dockerignore ├── .envrc.sample ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── 01-normal-issue.md │ ├── 02-bug_report.yml │ ├── bug_report.yml │ ├── config.yml │ ├── new_version.md │ └── normal-issue.md ├── PULL_REQUEST_TEMPLATE │ ├── pre-release-getting-started-script.md │ └── pre-release-rust-deps.md ├── actionlint.yaml ├── pull_request_template.md └── workflows │ ├── build.yml │ ├── general_daily_security.yml │ ├── integration-test.yml │ └── pr_pre-commit.yaml ├── .gitignore ├── .hadolint.yaml ├── .markdownlint.yaml ├── .pre-commit-config.yaml ├── .pylintrc ├── .readme ├── README.md.j2 ├── partials │ ├── borrowed │ │ ├── documentation.md.j2 │ │ ├── footer.md.j2 │ │ ├── header.md.j2 │ │ ├── links.md.j2 │ │ ├── overview_blurb.md.j2 │ │ └── related_reading.md.j2 │ └── main.md.j2 └── static │ └── borrowed │ ├── Icon_Stackable.svg │ ├── sdp_overview.png │ └── stackable_overview.png ├── .vscode ├── launch.json └── settings.json ├── .yamllint.yaml ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.nix ├── Cargo.toml ├── LICENSE ├── Makefile ├── README.md ├── Tiltfile ├── crate-hashes.json ├── default.nix ├── deny.toml ├── deploy ├── DO_NOT_EDIT.md ├── config-spec │ └── properties.yaml ├── helm │ ├── airflow-operator │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── README.md │ │ ├── configs │ │ │ └── properties.yaml │ │ ├── crds │ │ │ └── crds.yaml │ │ ├── templates │ │ │ ├── _helpers.tpl │ │ │ ├── _telemetry.tpl │ │ │ ├── configmap.yaml │ │ │ ├── deployment.yaml │ │ │ ├── roles.yaml │ │ │ └── serviceaccount.yaml │ │ └── values.yaml │ └── ct.yaml └── stackable-operators-ns.yaml ├── docker └── Dockerfile ├── docs ├── antora.yml ├── modules │ └── airflow │ │ ├── examples │ │ ├── example-airflow-dags-configmap.yaml │ │ ├── example-airflow-gitsync.yaml │ │ ├── example-airflow-incluster.yaml │ │ ├── example-airflow-kubernetes-executor-s3-logging.yaml │ │ ├── example-airflow-kubernetes-executor-s3-xcom.yaml │ │ ├── example-airflow-secret.yaml │ │ ├── example-airflow-spark-clusterrole.yaml │ │ ├── example-airflow-spark-clusterrolebinding.yaml │ │ ├── example-configmap.yaml │ │ ├── example-pyspark-pi.yaml │ │ ├── example-spark-dag.py │ │ ├── example_spark_kubernetes_operator.py │ │ ├── example_spark_kubernetes_sensor.py │ │ └── getting_started │ │ │ └── code │ │ │ ├── airflow-credentials.yaml │ │ │ ├── airflow.yaml │ │ │ ├── getting_started.sh │ │ │ ├── getting_started.sh.j2 │ │ │ ├── install_output.txt │ │ │ ├── install_output.txt.j2 │ │ │ ├── test_getting_started_helm.sh │ │ │ └── test_getting_started_stackablectl.sh │ │ ├── images │ │ ├── airflow_connection_ui.png │ │ ├── airflow_dag_graph.png │ │ ├── airflow_dag_log.png │ │ ├── airflow_dag_log_opensearch.png │ │ ├── airflow_dag_s3_logs.png │ │ ├── airflow_dags.png │ │ ├── airflow_edit_s3_connection.png │ │ ├── airflow_login.png │ │ ├── airflow_overview.drawio.svg │ │ ├── airflow_running.png │ │ ├── airflow_security.png │ │ ├── airflow_security_ldap.png │ │ └── getting_started │ │ │ ├── airflow_dags.png │ │ │ ├── airflow_login.png │ │ │ ├── airflow_pods.png │ │ │ └── airflow_running.png │ │ ├── pages │ │ ├── getting_started │ │ │ ├── first_steps.adoc │ │ │ ├── index.adoc │ │ │ └── installation.adoc │ │ ├── index.adoc │ │ ├── reference │ │ │ ├── commandline-parameters.adoc │ │ │ ├── crds.adoc │ │ │ ├── environment-variables.adoc │ │ │ └── index.adoc │ │ ├── required-external-components.adoc │ │ └── usage-guide │ │ │ ├── applying-custom-resources.adoc │ │ │ ├── index.adoc │ │ │ ├── listenerclass.adoc │ │ │ ├── logging.adoc │ │ │ ├── monitoring.adoc │ │ │ ├── mounting-dags.adoc │ │ │ ├── operations │ │ │ ├── cluster-operations.adoc │ │ │ ├── graceful-shutdown.adoc │ │ │ ├── index.adoc │ │ │ ├── pod-disruptions.adoc │ │ │ └── pod-placement.adoc │ │ │ ├── overrides.adoc │ │ │ ├── security.adoc │ │ │ ├── storage-resources.adoc │ │ │ └── using-kubernetes-executors.adoc │ │ └── partials │ │ ├── hardware-requirements.adoc │ │ ├── nav.adoc │ │ └── supported-versions.adoc └── templating_vars.yaml ├── examples ├── simple-airflow-cluster-dags-cmap.yaml ├── simple-airflow-cluster-ldap-insecure-tls.yaml ├── simple-airflow-cluster-ldap.yaml └── simple-airflow-cluster.yaml ├── nix ├── README.md ├── meta.json ├── sources.json └── sources.nix ├── renovate.json ├── rust-toolchain.toml ├── rust └── operator-binary │ ├── Cargo.toml │ ├── build.rs │ └── src │ ├── airflow_controller.rs │ ├── config.rs │ ├── controller_commons.rs │ ├── crd │ ├── affinity.rs │ ├── authentication.rs │ ├── authorization.rs │ └── mod.rs │ ├── env_vars.rs │ ├── main.rs │ ├── operations │ ├── graceful_shutdown.rs │ ├── mod.rs │ └── pdb.rs │ ├── product_logging.rs │ └── util.rs ├── rustfmt.toml ├── scripts ├── docs_templating.sh ├── ensure_one_trailing_newline.py ├── generate-manifests.sh ├── render_readme.sh ├── run-tests └── run_tests.sh ├── shell.nix └── tests ├── README-templating.md ├── infrastructure.yaml ├── kuttl-test.yaml.jinja2 ├── release.yaml ├── templates ├── .gitkeep └── kuttl │ ├── cluster-operation │ ├── 00-patch-ns.yaml.j2 │ ├── 02-assert.yaml │ ├── 02-install-postgresql.yaml │ ├── 04-assert.yaml │ ├── 04-install-redis.yaml │ ├── 06-assert.yaml.j2 │ ├── 06-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 08-assert.yaml │ ├── 08-install-airflow.yaml.j2 │ ├── 10-assert.yaml │ ├── 10-pause-airflow.yaml.j2 │ ├── 20-assert.yaml │ ├── 20-stop-airflow.yaml.j2 │ ├── 30-assert.yaml │ ├── 30-restart-airflow.yaml.j2 │ ├── helm-bitnami-postgresql-values.yaml.j2 │ └── helm-bitnami-redis-values.yaml.j2 │ ├── commons │ ├── health.py │ └── metrics.py │ ├── external-access │ ├── 00-patch-ns.yaml.j2 │ ├── 10-install-postgresql.yaml │ ├── 20-assert.yaml │ ├── 20-install-redis.yaml │ ├── 30-listener-classes.yaml │ ├── 40-assert.yaml │ ├── 40-install-airflow-cluster.yaml │ ├── helm-bitnami-postgresql-values.yaml.j2 │ ├── helm-bitnami-redis-values.yaml.j2 │ ├── install-airflow-cluster.yaml.j2 │ └── listener-classes.yaml │ ├── ldap │ ├── 00-patch-ns.yaml.j2 │ ├── 05-assert.yaml │ ├── 05-install-postgresql.yaml │ ├── 10-assert.yaml.j2 │ ├── 10-install-redis.yaml.j2 │ ├── 20-assert.yaml.j2 │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 30-assert.yaml │ ├── 30-install-openldap.yaml.j2 │ ├── 40-assert.yaml │ ├── 40-create-ldap-user.yaml │ ├── 50-create-authentication-classes.yaml.j2 │ ├── 60-assert.yaml.j2 │ ├── 60-install-airflow-cluster.yaml.j2 │ ├── 70-assert.yaml │ ├── 70-install-airflow-python.yaml │ ├── 80-assert.yaml │ ├── 80-health-check.yaml │ ├── 90-assert.yaml │ ├── 90-install-metrics-script.yaml │ ├── create_ldap_user.sh │ ├── helm-bitnami-postgresql-values.yaml.j2 │ └── helm-bitnami-redis-values.yaml.j2 │ ├── logging │ ├── 00-patch-ns.yaml.j2 │ ├── 05-assert.yaml │ ├── 05-install-postgresql.yaml │ ├── 10-assert.yaml │ ├── 10-install-redis.yaml │ ├── 20-assert.yaml.j2 │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 30-assert.yaml │ ├── 30-install-airflow-vector-aggregator.yaml │ ├── 40-create-configmap-with-prepared-logs.yaml │ ├── 41-assert.yaml.j2 │ ├── 41-install-airflow-cluster.yaml.j2 │ ├── 50-assert.yaml │ ├── 50-install-airflow-python.yaml │ ├── 51-assert.yaml │ ├── 51-health-check.yaml │ ├── 52-assert.yaml │ ├── 52-install-metrics-script.yaml │ ├── 60-assert.yaml │ ├── 60-test-log-aggregation.yaml │ ├── airflow-vector-aggregator-values.yaml.j2 │ ├── helm-bitnami-postgresql-values.yaml.j2 │ ├── helm-bitnami-redis-values.yaml.j2 │ ├── prepared-logs.py.json │ └── test_log_aggregation.py │ ├── mount-dags-configmap │ ├── 00-patch-ns.yaml.j2 │ ├── 05-assert.yaml │ ├── 05-install-postgresql.yaml │ ├── 10-assert.yaml.j2 │ ├── 10-install-redis.yaml.j2 │ ├── 20-assert.yaml.j2 │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 30-assert.yaml.j2 │ ├── 30-install-airflow-cluster.yaml.j2 │ ├── 40-assert.yaml │ ├── 40-install-airflow-python.yaml │ ├── 50-assert.yaml │ ├── 50-health-check.yaml │ ├── 60-assert.yaml │ ├── 60-install-metrics-script.yaml │ ├── helm-bitnami-postgresql-values.yaml.j2 │ └── helm-bitnami-redis-values.yaml.j2 │ ├── mount-dags-gitsync │ ├── 00-patch-ns.yaml.j2 │ ├── 03-assert.yaml │ ├── 03-install-postgresql.yaml │ ├── 04-assert.yaml.j2 │ ├── 04-install-redis.yaml.j2 │ ├── 05-assert.yaml.j2 │ ├── 05-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 10-clusterrole.yaml │ ├── 30-assert.yaml.j2 │ ├── 30-install-airflow-cluster.yaml.j2 │ ├── 31-assert.yaml.j2 │ ├── 40-assert.yaml │ ├── 40-install-airflow-python.yaml │ ├── 50-assert.yaml │ ├── 50-health-check.yaml │ ├── 60-assert.yaml │ ├── 60-install-metrics-script.yaml │ ├── dag_metrics.py │ ├── helm-bitnami-postgresql-values.yaml.j2 │ └── helm-bitnami-redis-values.yaml.j2 │ ├── oidc │ ├── 00-patch-ns.yaml.j2 │ ├── 10-assert.yaml │ ├── 10-install-postgresql.yaml │ ├── 20-assert.yaml.j2 │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 30-assert.yaml │ ├── 30-install-keycloak.yaml │ ├── 40-assert.yaml │ ├── 40-install-airflow.yaml │ ├── 50-assert.yaml │ ├── 50-install-test-container.yaml.j2 │ ├── 60-assert.yaml │ ├── 60-login.yaml │ ├── helm-bitnami-postgresql-values.yaml.j2 │ ├── install-airflow.yaml.j2 │ ├── install-keycloak.yaml.j2 │ └── login.py │ ├── opa │ ├── 10-patch-ns.yaml.j2 │ ├── 11-assert.yaml │ ├── 11-install-postgresql.yaml │ ├── 11_helm-bitnami-postgresql-values.yaml.j2 │ ├── 12-assert.yaml.j2 │ ├── 12-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 20-assert.yaml │ ├── 20-install-opa.yaml.j2 │ ├── 30-assert.yaml │ ├── 30-install-airflow.yaml.j2 │ ├── 31-opa-rules.yaml │ ├── 40-assert.yaml │ ├── 40-install-test-container.yaml.j2 │ ├── 41-assert.yaml │ ├── 41-check-authorization.yaml │ └── 41_check-authorization.py │ ├── orphaned-resources │ ├── 00-patch-ns.yaml.j2 │ ├── 05-assert.yaml │ ├── 05-install-postgresql.yaml │ ├── 10-assert.yaml │ ├── 10-install-redis.yaml │ ├── 20-assert.yaml.j2 │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 30-assert.yaml │ ├── 30-install-airflow-cluster.yaml.j2 │ ├── 40-assert.yaml │ ├── 40-errors.yaml │ ├── 40-remove-webserver.yaml │ ├── 50-assert.yaml │ ├── 50-change-worker-rolegroup.yaml │ ├── 50-errors.yaml │ ├── helm-bitnami-postgresql-values.yaml.j2 │ └── helm-bitnami-redis-values.yaml.j2 │ ├── overrides │ ├── 05-assert.yaml │ ├── 05-install-postgresql.yaml │ ├── 07-assert.yaml.j2 │ ├── 07-install-redis.yaml.j2 │ ├── 10-assert.yaml │ ├── 10-install-airflow.yaml.j2 │ ├── 11-assert.yaml │ ├── 20-assert.yaml │ ├── 20-install-airflow2.yaml.j2 │ ├── 21-assert.yaml │ ├── helm-bitnami-postgresql-values.yaml.j2 │ └── helm-bitnami-redis-values.yaml.j2 │ ├── resources │ ├── 00-patch-ns.yaml.j2 │ ├── 05-assert.yaml │ ├── 05-install-postgresql.yaml │ ├── 10-assert.yaml │ ├── 10-install-redis.yaml │ ├── 20-assert.yaml.j2 │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 30-assert.yaml.j2 │ ├── 30-install-airflow-cluster.yaml.j2 │ ├── helm-bitnami-postgresql-values.yaml.j2 │ └── helm-bitnami-redis-values.yaml.j2 │ └── smoke │ ├── 00-patch-ns.yaml.j2 │ ├── 00-range-limit.yaml │ ├── 10-assert.yaml │ ├── 10-install-postgresql.yaml │ ├── 20-assert.yaml.j2 │ ├── 20-install-redis.yaml.j2 │ ├── 30-assert.yaml.j2 │ ├── 30-install-vector-aggregator-discovery-configmap.yaml.j2 │ ├── 40-assert.yaml.j2 │ ├── 40-install-airflow-cluster.yaml.j2 │ ├── 41-assert.yaml │ ├── 42-assert.yaml │ ├── 50-assert.yaml │ ├── 50-install-airflow-python.yaml │ ├── 60-assert.yaml │ ├── 60-health-check.yaml │ ├── 70-assert.yaml │ ├── 70-install-metrics-script.yaml │ ├── helm-bitnami-postgresql-values.yaml.j2 │ └── helm-bitnami-redis-values.yaml.j2 └── test-definition.yaml /.actionlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | self-hosted-runner: 3 | # Ubicloud machines we are using 4 | labels: 5 | - ubicloud-standard-8-arm 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | debug/ 2 | target/ 3 | **/*.rs.bk 4 | 5 | .idea/ 6 | *.iws 7 | 8 | Cargo.nix 9 | crate-hashes.json 10 | result 11 | image.tar 12 | 13 | # We do NOT want to ignore .git because we use the `built` crate to gather the current git commit hash at built time 14 | # This means we need the .git directory in our Docker image, it will be thrown away and won't be included in the final image 15 | -------------------------------------------------------------------------------- /.envrc.sample: -------------------------------------------------------------------------------- 1 | # vim: syntax=conf 2 | # 3 | # If you use direnv, you can autoload the nix shell: 4 | # You will need to allow the directory the first time. 5 | use nix 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | nix/** linguist-generated 2 | Cargo.nix linguist-generated 3 | crate-hashes.json linguist-generated 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/01-normal-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Normal issue 3 | about: This is just a normal empty issue with a simple checklist 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Issue checklist 11 | 12 | This is a simple checklist of things to bear in mind when creating a new issue. 13 | 14 | - [ ] **Describe the use-case**: As far as possible, use the pattern "As a [type of user], I would like [feature/functionality] to be able to do [specific action]." This helps identify the feature and the problem it addresses. 15 | - [ ] **Indicate importance and urgency**: Use a scale (e.g., low, medium, high) to indicate the level of importance and urgency. 16 | - [ ] **Work-around**: If there is a known work-around, describe it briefly. 17 | - [ ] **Environment**: Describe the environment where the issue occurs (e.g., SDP version, K8S version, etc.). 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/02-bug_report.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "🐛 Bug Report" 3 | description: "If something isn't working as expected 🤔." 4 | labels: ["type/bug"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: Thanks for taking the time to file a bug report! Please fill out this form as completely as possible. 9 | 10 | - type: input 11 | attributes: 12 | label: Affected Stackable version 13 | description: Which version of the Stackable Operator do you see this bug in? 14 | 15 | # - type: input 16 | attributes: 17 | label: Affected Apache Airflow version 18 | description: Which version of Apache Airflow do you see this bug in? 19 | # 20 | - type: textarea 21 | attributes: 22 | label: Current and expected behavior 23 | description: A clear and concise description of what the operator is doing and what you would expect. 24 | validations: 25 | required: true 26 | 27 | - type: textarea 28 | attributes: 29 | label: Possible solution 30 | description: "If you have suggestions on a fix for the bug." 31 | 32 | - type: textarea 33 | attributes: 34 | label: Additional context 35 | description: "Add any other context about the problem here. Or a screenshot if applicable." 36 | 37 | - type: textarea 38 | attributes: 39 | label: Environment 40 | description: | 41 | What type of kubernetes cluster you are running aginst (k3s/eks/aks/gke/other) and any other information about your environment? 42 | placeholder: | 43 | Examples: 44 | Output of `kubectl version --short` 45 | 46 | - type: dropdown 47 | attributes: 48 | label: Would you like to work on fixing this bug? 49 | description: | 50 | **NOTE**: Let us know if you would like to submit a PR for this. We are more than happy to help you through the process. 51 | options: 52 | - "yes" 53 | - "no" 54 | - "maybe" 55 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "🐛 Bug Report" 3 | description: "If something isn't working as expected 🤔." 4 | labels: ["type/bug"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: Thanks for taking the time to file a bug report! Please fill out this form as completely as possible. 9 | 10 | - type: input 11 | attributes: 12 | label: Affected Stackable version 13 | description: Which version of the Stackable Operator do you see this bug in? 14 | 15 | - type: input 16 | attributes: 17 | label: Affected Apache Airflow version 18 | description: Which version of Apache Airflow do you see this bug in? 19 | 20 | - type: textarea 21 | attributes: 22 | label: Current and expected behavior 23 | description: A clear and concise description of what the operator is doing and what you would expect. 24 | validations: 25 | required: true 26 | 27 | - type: textarea 28 | attributes: 29 | label: Possible solution 30 | description: "If you have suggestions on a fix for the bug." 31 | 32 | - type: textarea 33 | attributes: 34 | label: Additional context 35 | description: "Add any other context about the problem here. Or a screenshot if applicable." 36 | 37 | - type: textarea 38 | attributes: 39 | label: Environment 40 | description: | 41 | What type of kubernetes cluster you are running aginst (k3s/eks/aks/gke/other) and any other information about your environment? 42 | placeholder: | 43 | Examples: 44 | Output of `kubectl version --short` 45 | 46 | - type: dropdown 47 | attributes: 48 | label: Would you like to work on fixing this bug? 49 | description: | 50 | **NOTE**: Let us know if you would like to submit a PR for this. We are more than happy to help you through the process. 51 | options: 52 | - "yes" 53 | - "no" 54 | - "maybe" 55 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | blank_issues_enabled: true 3 | contact_links: 4 | - name: 🙋🏾 Question 5 | about: Use this to ask a question about this project 6 | url: https://github.com/orgs/stackabletech/discussions/new?category=q-a 7 | - name: 🚀 Feature Requests and other things 8 | about: Open an issue with your feature request or any other issue not covered elsewhere 9 | url: https://github.com/stackabletech/airflow-operator/issues/new 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/new_version.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: New Version 3 | about: Request support for a new product version 4 | title: "[NEW VERSION]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Which new version of Apache Airflow should we support? 11 | 12 | Please specify the version, version range or version numbers to support, please also add these to the issue title 13 | 14 | ## Additional information 15 | 16 | If possible, provide a link to release notes/changelog 17 | 18 | ## Changes required 19 | 20 | Are there any upstream changes that we need to support? 21 | e.g. new features, changed features, deprecated features etc. 22 | 23 | ## Implementation checklist 24 | 25 | 29 | 30 | - [ ] Update the Docker image 31 | - [ ] Update documentation to include supported version(s) 32 | - [ ] Update and test getting started guide with updated version(s) 33 | - [ ] Update operator to support the new version (if needed) 34 | - [ ] Update integration tests to test use the new versions (in addition or replacing old versions 35 | - [ ] Update examples to use new versions 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/normal-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Normal issue 3 | about: This is just a normal empty issue with a simple checklist 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Issue checklist 11 | 12 | This is a simple checklist of things to bear in mind when creating a new issue. 13 | 14 | - [ ] Describe the use-case, as far is possible. For instance, using the pattern "As a XXXX, I would like XXXX to be able to do XXXX" helps to identify the feature as well as the problem it is intended to address. 15 | - [ ] Indicate an approximate level of importance and urgency. 16 | - [ ] Indicate if there is a known work-around until such time as the issue has been implemented. 17 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pre-release-getting-started-script.md: -------------------------------------------------------------------------------- 1 | ## Check and Update Getting Started Script 2 | 3 | 7 | 8 | 11 | 12 | Part of 13 | 14 | > [!NOTE] 15 | > During a Stackable release we need to check (and optionally update) the 16 | > getting-started scripts to ensure they still work after product and operator 17 | > updates. 18 | 19 | ```shell 20 | # Some of the scripts are in a code/ subdirectory 21 | # pushd docs/modules/superset/examples/getting_started 22 | # pushd docs/modules/superset/examples/getting_started/code 23 | pushd $(fd -td getting_started | grep examples); cd code 2>/dev/null || true 24 | 25 | # Make a fresh cluster (~12 seconds) 26 | kind delete cluster && kind create cluster 27 | ./getting_started.sh stackablectl 28 | 29 | # Make a fresh cluster (~12 seconds) 30 | kind delete cluster && kind create cluster 31 | ./getting_started.sh helm 32 | 33 | popd 34 | ``` 35 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pre-release-rust-deps.md: -------------------------------------------------------------------------------- 1 | ## Bump Rust Dependencies for Stackable Release YY.M.X 2 | 3 | 7 | 8 | 11 | 12 | Part of 13 | 14 | > [!NOTE] 15 | > During a Stackable release we need to update various Rust dependencies before 16 | > entering the final release period to ensure we run the latest versions of 17 | > crates. These bumps also include previously updated and released crates from 18 | > the `operator-rs` repository. 19 | 20 | ### Tasks 21 | 22 | - [ ] Bump Rust Dependencies, see below for more details. 23 | - [ ] Add changelog entry stating which important crates were bumped (including the version). 24 | 25 | > [!NOTE] 26 | > The bumping / updating of Rust dependencies is done in multiple steps: 27 | > 28 | > 1. Update the minimum Version in the root `Cargo.toml` manifest. 29 | > 2. Run the `cargo update` command, which also updates the `Cargo.lock` file. 30 | > 3. Lastly, run `make regenerate-nix` to update the `Cargo.nix` file. 31 | 32 | ### Bump Rust Dependencies 33 | 34 | - [ ] Bump `stackable-operator` and friends 35 | - [ ] Bump `product-config` 36 | - [ ] Bump all other dependencies 37 | -------------------------------------------------------------------------------- /.github/actionlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | self-hosted-runner: 3 | # Ubicloud machines we are using 4 | labels: 5 | - ubicloud-standard-8-arm 6 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | *Please add a description here. This will become the commit message of the merge request later.* 4 | 5 | ## Definition of Done Checklist 6 | 7 | - Not all of these items are applicable to all PRs, the author should update this template to only leave the boxes in that are relevant 8 | - Please make sure all these things are done and tick the boxes 9 | 10 | ### Author 11 | 12 | - [ ] Changes are OpenShift compatible 13 | - [ ] CRD changes approved 14 | - [ ] CRD documentation for all fields, following the [style guide](https://docs.stackable.tech/home/nightly/contributor/docs/style-guide). 15 | - [ ] Helm chart can be installed and deployed operator works 16 | - [ ] Integration tests passed (for non trivial changes) 17 | - [ ] Changes need to be "offline" compatible 18 | - [ ] Links to generated (nightly) docs added 19 | - [ ] Release note snippet added 20 | 21 | ### Reviewer 22 | 23 | - [ ] Code contains useful comments 24 | - [ ] Code contains useful logging statements 25 | - [ ] (Integration-)Test cases added 26 | - [ ] Documentation added or updated. Follows the [style guide](https://docs.stackable.tech/home/nightly/contributor/docs/style-guide). 27 | - [ ] Changelog updated 28 | - [ ] Cargo.toml only contains references to git tags (not specific commits or branches) 29 | 30 | ### Acceptance 31 | 32 | - [ ] Feature Tracker has been updated 33 | - [ ] Proper release label has been added 34 | - [ ] Links to generated (nightly) docs added 35 | - [ ] Release note snippet added 36 | - [ ] Add `type/deprecation` label & add to the [deprecation schedule](https://github.com/orgs/stackabletech/projects/44/views/1) 37 | - [ ] Add `type/experimental` label & add to the [experimental features tracker](https://github.com/orgs/stackabletech/projects/47) 38 | -------------------------------------------------------------------------------- /.github/workflows/general_daily_security.yml: -------------------------------------------------------------------------------- 1 | # ============= 2 | # This file is automatically generated from the templates in stackabletech/operator-templating 3 | # DON'T MANUALLY EDIT THIS FILE 4 | # ============= 5 | --- 6 | name: Daily Security Audit 7 | 8 | on: 9 | schedule: 10 | - cron: '15 4 * * *' 11 | workflow_dispatch: 12 | 13 | permissions: {} 14 | 15 | jobs: 16 | audit: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 20 | with: 21 | persist-credentials: false 22 | - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 23 | with: 24 | token: ${{ secrets.GITHUB_TOKEN }} 25 | -------------------------------------------------------------------------------- /.github/workflows/pr_pre-commit.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: pre-commit 3 | 4 | on: 5 | pull_request: 6 | merge_group: 7 | 8 | env: 9 | CARGO_TERM_COLOR: always 10 | NIX_PKG_MANAGER_VERSION: "2.28.3" 11 | RUST_TOOLCHAIN_VERSION: "nightly-2025-05-26" 12 | HADOLINT_VERSION: "v2.12.0" 13 | PYTHON_VERSION: "3.12" 14 | 15 | jobs: 16 | pre-commit: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Install host dependencies 20 | uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3 21 | with: 22 | packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https 23 | version: ubuntu-latest 24 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 25 | with: 26 | persist-credentials: false 27 | submodules: recursive 28 | fetch-depth: 0 29 | - uses: stackabletech/actions/run-pre-commit@9aae2d1c14239021bfa33c041010f6fb7adec815 # v0.8.2 30 | with: 31 | python-version: ${{ env.PYTHON_VERSION }} 32 | rust: ${{ env.RUST_TOOLCHAIN_VERSION }} 33 | hadolint: ${{ env.HADOLINT_VERSION }} 34 | nix: ${{ env.NIX_PKG_MANAGER_VERSION }} 35 | nix-github-token: ${{ secrets.GITHUB_TOKEN }} 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tests/_work/ 2 | debug/ 3 | target/ 4 | **/*.rs.bk 5 | 6 | .idea/ 7 | *.iws 8 | *.iml 9 | 10 | *.tgz 11 | 12 | result 13 | image.tar 14 | 15 | tilt_options.json 16 | 17 | .direnv/ 18 | .direnvrc 19 | .envrc 20 | 21 | .DS_Store 22 | -------------------------------------------------------------------------------- /.hadolint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ignored: 3 | # Warning: Use the -y switch to avoid manual input dnf install -y 4 | # https://github.com/hadolint/hadolint/wiki/DL3038 5 | # Reason: We set `assumeyes=True` in dnf.conf in our base image 6 | - DL3038 7 | 8 | # Warning: Specify version with dnf install -y - 9 | # https://github.com/hadolint/hadolint/wiki/DL3041 10 | # Reason: It's good advice, but we're not set up to pin versions just yet 11 | - DL3041 12 | -------------------------------------------------------------------------------- /.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # All defaults or options can be checked here: 3 | # https://github.com/DavidAnson/markdownlint/blob/main/schema/.markdownlint.yaml 4 | 5 | # Default state for all rules 6 | default: true 7 | 8 | # MD013/line-length - Line length 9 | MD013: 10 | # Number of characters 11 | line_length: 9999 12 | # Number of characters for headings 13 | heading_line_length: 9999 14 | # Number of characters for code blocks 15 | code_block_line_length: 9999 16 | 17 | # MD033/no-inline-html 18 | MD033: 19 | allowed_elements: [h1, img, p] 20 | 21 | # MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the same content 22 | MD024: 23 | # Only check sibling headings 24 | siblings_only: true 25 | 26 | # MD041/first-line-heading/first-line-h1 First line in a file should be a top-level heading 27 | MD041: false # Github issues and PRs already have titles, and H1 is enormous in the description box. 28 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | 3 | # These rules are for missing docstrings which doesn't matter much for most of our simple scripts 4 | disable=C0114,C0115,C0116 5 | 6 | [FORMAT] 7 | 8 | max-line-length=999 9 | indent-string=' ' 10 | -------------------------------------------------------------------------------- /.readme/README.md.j2: -------------------------------------------------------------------------------- 1 | {%- set title="Stackable Operator for Apache Airflow" -%} 2 | {%- set operator_name="airflow" -%} 3 | {%- set operator_docs_slug="airflow" -%} 4 | {%- set related_reading_links=[] -%} 5 | 6 | {% filter trim %} 7 | {%- include "partials/borrowed/header.md.j2" -%} 8 | {% endfilter %} 9 | 10 | {% filter trim %} 11 | {%- include "partials/borrowed/links.md.j2" -%} 12 | {% endfilter %} 13 | 14 | {% filter trim %} 15 | {%- include "partials/main.md.j2" -%} 16 | {% endfilter %} 17 | 18 | {% filter trim %} 19 | {%- include "partials/borrowed/footer.md.j2" -%} 20 | {% endfilter %} 21 | 22 | {% filter trim %} 23 | {%- include "partials/borrowed/related_reading.md.j2" -%} 24 | {% endfilter %} 25 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/documentation.md.j2: -------------------------------------------------------------------------------- 1 | 2 | ## Documentation 3 | 4 | The stable documentation for this operator can be found in our [Stackable Data Platform documentation](https://docs.stackable.tech/home/stable/{{operator_docs_slug}}). 5 | If you are interested in the most recent state of this repository, check out the [nightly docs](https://docs.stackable.tech/home/nightly/{{operator_docs_slug}}) instead. 6 | 7 | The documentation for all Stackable products can be found at [docs.stackable.tech](https://docs.stackable.tech). 8 | 9 | If you have a question about the Stackable Data Platform, contact us via our [homepage](https://stackable.tech/) or ask a public question in our [Discussions forum](https://github.com/orgs/stackabletech/discussions). 10 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/header.md.j2: -------------------------------------------------------------------------------- 1 | 2 |

3 | Stackable Logo 4 |

5 | 6 |

{{title}}

7 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/links.md.j2: -------------------------------------------------------------------------------- 1 | 2 | [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/stackabletech/{{operator_name}}-operator/graphs/commit-activity) 3 | [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-green.svg)](https://docs.stackable.tech/home/stable/contributor/index.html) 4 | [![License OSL3.0](https://img.shields.io/badge/license-OSL3.0-green)](./LICENSE) 5 | 6 | [Documentation](https://docs.stackable.tech/home/stable/{{operator_docs_slug}}) {% if quickstart_link %}| [Quickstart]({{quickstart_link}}) {% endif %}| [Stackable Data Platform](https://stackable.tech/) | [Platform Docs](https://docs.stackable.tech/) | [Discussions](https://github.com/orgs/stackabletech/discussions) | [Discord](https://discord.gg/7kZ3BNnCAF) 7 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/overview_blurb.md.j2: -------------------------------------------------------------------------------- 1 | 2 | It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka, Apache Druid, Trino or Apache Spark, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms). 3 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/related_reading.md.j2: -------------------------------------------------------------------------------- 1 | 2 | {%- if related_reading_links -%} 3 | ## Related Reading 4 | {% for (text, link) in related_reading_links %} 5 | * [{{text}}]({{link}}) 6 | {%- endfor %} 7 | {%- endif -%} 8 | -------------------------------------------------------------------------------- /.readme/partials/main.md.j2: -------------------------------------------------------------------------------- 1 | This is a Kubernetes operator to manage [Apache Airflow](https://airflow.apache.org/) ensembles. 2 | 3 | {% filter trim %} 4 | {%- include "partials/borrowed/overview_blurb.md.j2" -%} 5 | {% endfilter %} 6 | 7 | ## Installation 8 | 9 | You can install the operator using [stackablectl or helm](https://docs.stackable.tech/home/stable/{{operator_name}}/getting_started/installation). 10 | 11 | Read on to get started with it, or see it in action in one of our [demos](https://stackable.tech/en/demos/). 12 | 13 | ## Getting Started 14 | 15 | You can follow this [tutorial](https://docs.stackable.tech/home/stable/{{operator_name}}/getting_started/first_steps). 16 | 17 | {% filter trim %} 18 | {%- include "partials/borrowed/documentation.md.j2" -%} 19 | {% endfilter %} 20 | -------------------------------------------------------------------------------- /.readme/static/borrowed/Icon_Stackable.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /.readme/static/borrowed/sdp_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/.readme/static/borrowed/sdp_overview.png -------------------------------------------------------------------------------- /.readme/static/borrowed/stackable_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/.readme/static/borrowed/stackable_overview.png -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "lldb", 6 | "request": "launch", 7 | "name": "Debug operator binary", 8 | "cargo": { 9 | "args": ["build"], 10 | "filter": { 11 | "name": "stackable-{[ operator.name }]", 12 | "kind": "bin" 13 | } 14 | }, 15 | "args": ["run"], 16 | "cwd": "${workspaceFolder}" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rust-analyzer.rustfmt.overrideCommand": [ 3 | "rustfmt", 4 | "+nightly-2025-05-26", 5 | "--edition", 6 | "2024", 7 | "--" 8 | ], 9 | } 10 | -------------------------------------------------------------------------------- /.yamllint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | ignore: | 5 | deploy/helm/**/templates 6 | 7 | rules: 8 | line-length: disable 9 | truthy: 10 | check-keys: false 11 | comments: 12 | min-spaces-from-content: 1 # Needed due to https://github.com/adrienverge/yamllint/issues/443 13 | indentation: 14 | indent-sequences: consistent 15 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["rust/operator-binary"] 3 | resolver = "2" 4 | 5 | [workspace.package] 6 | version = "0.0.0-dev" 7 | authors = ["Stackable GmbH "] 8 | license = "OSL-3.0" 9 | edition = "2021" 10 | repository = "https://github.com/stackabletech/airflow-operator" 11 | 12 | [workspace.dependencies] 13 | product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } 14 | stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.93.1" } 15 | 16 | anyhow = "1.0" 17 | built = { version = "0.7", features = ["chrono", "git2"] } 18 | clap = "4.5" 19 | const_format = "0.2" 20 | fnv = "1.0" 21 | futures = { version = "0.3", features = ["compat"] } 22 | indoc = "2.0" 23 | rstest = "0.25" 24 | semver = "1.0" 25 | serde = { version = "1.0", features = ["derive"] } 26 | serde_json = "1.0" 27 | serde_yaml = "0.9" 28 | snafu = "0.8" 29 | strum = { version = "0.27", features = ["derive"] } 30 | tokio = { version = "1.40", features = ["full"] } 31 | tracing = "0.1" 32 | 33 | # [patch."https://github.com/stackabletech/operator-rs.git"] 34 | # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } 35 | # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } 36 | -------------------------------------------------------------------------------- /Tiltfile: -------------------------------------------------------------------------------- 1 | # If tilt_options.json exists read it and load the default_registry value from it 2 | settings = read_json('tilt_options.json', default={}) 3 | registry = settings.get('default_registry', 'oci.stackable.tech/sandbox') 4 | 5 | # Configure default registry either read from config file above, or with default value of "oci.stackable.tech/sandbox" 6 | default_registry(registry) 7 | 8 | meta = read_json('nix/meta.json') 9 | operator_name = meta['operator']['name'] 10 | 11 | custom_build( 12 | registry + '/' + operator_name, 13 | 'make regenerate-nix && nix-build . -A docker --argstr dockerName "${EXPECTED_REGISTRY}/' + operator_name + '" && ./result/load-image | docker load', 14 | deps=['rust', 'Cargo.toml', 'Cargo.lock', 'default.nix', "nix", 'build.rs', 'vendor'], 15 | ignore=['*.~undo-tree~'], 16 | # ignore=['result*', 'Cargo.nix', 'target', *.yaml], 17 | outputs_image_ref_to='result/ref', 18 | ) 19 | 20 | # Load the latest CRDs from Nix 21 | watch_file('result') 22 | if os.path.exists('result'): 23 | k8s_yaml('result/crds.yaml') 24 | 25 | # We need to set the correct image annotation on the operator Deployment to use e.g. 26 | # oci.stackable.tech/sandbox/opa-operator:7y19m3d8clwxlv34v5q2x4p7v536s00g instead of 27 | # oci.stackable.tech/sandbox/opa-operator:0.0.0-dev (which does not exist) 28 | k8s_kind('Deployment', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') 29 | 30 | # Exclude stale CRDs from Helm chart, and apply the rest 31 | helm_crds, helm_non_crds = filter_yaml( 32 | helm( 33 | 'deploy/helm/' + operator_name, 34 | name=operator_name, 35 | namespace="stackable-operators", 36 | set=[ 37 | 'image.repository=' + registry + '/' + operator_name, 38 | ], 39 | ), 40 | api_version = "^apiextensions\\.k8s\\.io/.*$", 41 | kind = "^CustomResourceDefinition$", 42 | ) 43 | k8s_yaml(helm_non_crds) 44 | -------------------------------------------------------------------------------- /crate-hashes.json: -------------------------------------------------------------------------------- 1 | { 2 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#k8s-version@0.1.3": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", 3 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-operator-derive@0.3.1": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", 4 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-operator@0.93.1": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", 5 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-shared@0.0.1": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", 6 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-telemetry@0.6.0": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", 7 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-versioned-macros@0.7.1": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", 8 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-versioned@0.7.1": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", 9 | "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" 10 | } -------------------------------------------------------------------------------- /deploy/DO_NOT_EDIT.md: -------------------------------------------------------------------------------- 1 | # DO NOT EDIT 2 | 3 | These Helm charts and manifests are automatically generated. 4 | Please do not edit anything except for files explicitly mentioned below in this 5 | directory manually. 6 | 7 | The following files are ok to edit: 8 | 9 | - helm/airflow-operator/templates/roles.yaml 10 | - helm/airflow-operator/values.yaml 11 | 12 | The details are in-motion but check this repository for a few details: 13 | 14 | -------------------------------------------------------------------------------- /deploy/config-spec/properties.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | spec: 3 | units: [] 4 | properties: 5 | - property: &credentialsSecret 6 | propertyNames: 7 | - name: "credentialsSecret" 8 | kind: 9 | type: "env" 10 | datatype: 11 | type: "string" 12 | roles: 13 | - name: "node" 14 | required: true 15 | asOfVersion: "0.0.0" 16 | description: "The secret where the Airflow credentials are stored." 17 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # ============= 2 | # This file is automatically generated from the templates in stackabletech/operator-templating 3 | # DON'T MANUALLY EDIT THIS FILE 4 | # ============= 5 | 6 | # Patterns to ignore when building packages. 7 | # This supports shell glob matching, relative path matching, and 8 | # negation (prefixed with !). Only one pattern per line. 9 | .DS_Store 10 | # Common VCS dirs 11 | .git/ 12 | .gitignore 13 | .bzr/ 14 | .bzrignore 15 | .hg/ 16 | .hgignore 17 | .svn/ 18 | # Common backup files 19 | *.swp 20 | *.bak 21 | *.tmp 22 | *.orig 23 | *~ 24 | # Various IDEs 25 | .project 26 | .idea/ 27 | *.tmproj 28 | .vscode/ 29 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v2 3 | name: airflow-operator 4 | version: "0.0.0-dev" 5 | appVersion: "0.0.0-dev" 6 | description: The Stackable Operator for Apache Airflow 7 | home: https://github.com/stackabletech/airflow-operator 8 | maintainers: 9 | - name: Stackable 10 | url: https://www.stackable.tech 11 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Helm Chart for Stackable Operator for Apache Airflow 3 | 4 | This Helm Chart can be used to install Custom Resource Definitions and the Operator for Apache Airflow provided by Stackable. 5 | 6 | ## Requirements 7 | 8 | - Create a [Kubernetes Cluster](../Readme.md) 9 | - Install [Helm](https://helm.sh/docs/intro/install/) 10 | 11 | ## Install the Stackable Operator for Apache Airflow 12 | 13 | ```bash 14 | # From the root of the operator repository 15 | make compile-chart 16 | 17 | helm install airflow-operator deploy/helm/airflow-operator 18 | ``` 19 | 20 | ## Usage of the CRDs 21 | 22 | The usage of this operator and its CRDs is described in the [documentation](https://docs.stackable.tech/airflow/index.html) 23 | 24 | The operator has example requests included in the [`/examples`](https://github.com/stackabletech/airflow-operator/tree/main/examples) directory. 25 | 26 | ## Links 27 | 28 | 29 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/configs/properties.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | spec: 3 | units: [] 4 | properties: 5 | - property: &credentialsSecret 6 | propertyNames: 7 | - name: "credentialsSecret" 8 | kind: 9 | type: "env" 10 | datatype: 11 | type: "string" 12 | roles: 13 | - name: "node" 14 | required: true 15 | asOfVersion: "0.0.0" 16 | description: "The secret where the Airflow credentials are stored." 17 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/templates/_telemetry.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Create a list of telemetry related env vars. 3 | */}} 4 | {{- define "telemetry.envVars" -}} 5 | {{- with .Values.telemetry }} 6 | {{- if not .consoleLog.enabled }} 7 | - name: CONSOLE_LOG_DISABLED 8 | value: "true" 9 | {{- end }} 10 | {{- if and .consoleLog.enabled .consoleLog.level }} 11 | - name: CONSOLE_LOG_LEVEL 12 | value: {{ .consoleLog.level }} 13 | {{ end }} 14 | {{- if and .consoleLog.enabled .consoleLog.format }} 15 | - name: CONSOLE_LOG_FORMAT 16 | value: {{ .consoleLog.format }} 17 | {{ end }} 18 | {{- if .fileLog.enabled }} 19 | - name: FILE_LOG_DIRECTORY 20 | value: /stackable/logs/{{ include "operator.appname" $ }} 21 | {{- end }} 22 | {{- if and .fileLog.enabled .fileLog.level }} 23 | - name: FILE_LOG_LEVEL 24 | value: {{ .fileLog.level }} 25 | {{- end }} 26 | {{- if and .fileLog.enabled .fileLog.rotationPeriod }} 27 | - name: FILE_LOG_ROTATION_PERIOD 28 | value: {{ .fileLog.rotationPeriod }} 29 | {{- end }} 30 | {{- if and .fileLog.enabled .fileLog.maxFiles }} 31 | - name: FILE_LOG_MAX_FILES 32 | value: {{ quote .fileLog.maxFiles }} 33 | {{- end }} 34 | {{- if .otelLogExporter.enabled }} 35 | - name: OTEL_LOG_EXPORTER_ENABLED 36 | value: "true" 37 | {{- end }} 38 | {{- if and .otelLogExporter.enabled .otelLogExporter.level }} 39 | - name: OTEL_LOG_EXPORTER_LEVEL 40 | value: {{ .otelLogExporter.level }} 41 | {{- end }} 42 | {{- if and .otelLogExporter.enabled .otelLogExporter.endpoint }} 43 | - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT 44 | value: {{ .otelLogExporter.endpoint }} 45 | {{- end }} 46 | {{- if .otelTraceExporter.enabled }} 47 | - name: OTEL_TRACE_EXPORTER_ENABLED 48 | value: "true" 49 | {{- end }} 50 | {{- if and .otelTraceExporter.enabled .otelTraceExporter.level }} 51 | - name: OTEL_TRACE_EXPORTER_LEVEL 52 | value: {{ .otelTraceExporter.level }} 53 | {{- end }} 54 | {{- if and .otelTraceExporter.enabled .otelTraceExporter.endpoint }} 55 | - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT 56 | value: {{ .otelTraceExporter.endpoint }} 57 | {{- end }} 58 | {{- end }} 59 | {{- end }} 60 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | data: 4 | {{ (.Files.Glob "configs/*").AsConfig | indent 2 }} 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ include "operator.fullname" . }}-configmap 8 | labels: 9 | {{- include "operator.labels" . | nindent 4 }} 10 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | {{ if .Values.serviceAccount.create -}} 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: {{ include "operator.fullname" . }}-serviceaccount 7 | labels: 8 | {{- include "operator.labels" . | nindent 4 }} 9 | {{- with .Values.serviceAccount.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | # This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. 16 | kind: ClusterRoleBinding 17 | metadata: 18 | name: {{ include "operator.fullname" . }}-clusterrolebinding 19 | labels: 20 | {{- include "operator.labels" . | nindent 4 }} 21 | subjects: 22 | - kind: ServiceAccount 23 | name: {{ include "operator.fullname" . }}-serviceaccount 24 | namespace: {{ .Release.Namespace }} 25 | roleRef: 26 | kind: ClusterRole 27 | name: {{ include "operator.fullname" . }}-clusterrole 28 | apiGroup: rbac.authorization.k8s.io 29 | {{- end }} 30 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for airflow-operator. 2 | --- 3 | image: 4 | repository: oci.stackable.tech/sdp/airflow-operator 5 | pullPolicy: IfNotPresent 6 | pullSecrets: [] 7 | 8 | nameOverride: "" 9 | fullnameOverride: "" 10 | 11 | serviceAccount: 12 | # Specifies whether a service account should be created 13 | create: true 14 | # Annotations to add to the service account 15 | annotations: {} 16 | # The name of the service account to use. 17 | # If not set and create is true, a name is generated using the fullname template 18 | name: "" 19 | 20 | # Provide additional labels which get attached to all deployed resources 21 | labels: 22 | stackable.tech/vendor: Stackable 23 | 24 | podAnnotations: {} 25 | 26 | podSecurityContext: {} 27 | # fsGroup: 2000 28 | 29 | securityContext: {} 30 | 31 | resources: 32 | limits: 33 | cpu: 100m 34 | memory: 128Mi 35 | requests: 36 | cpu: 100m 37 | memory: 128Mi 38 | 39 | nodeSelector: {} 40 | 41 | tolerations: [] 42 | 43 | affinity: {} 44 | 45 | # When running on a non-default Kubernetes cluster domain, the cluster domain can be configured here. 46 | # See the https://docs.stackable.tech/home/stable/guides/kubernetes-cluster-domain guide for details. 47 | # kubernetesClusterDomain: my-cluster.local 48 | 49 | # See all available options and detailed explanations about the concept here: 50 | # https://docs.stackable.tech/home/stable/concepts/telemetry/ 51 | telemetry: 52 | consoleLog: 53 | enabled: true 54 | fileLog: 55 | enabled: false 56 | rotationPeriod: hourly 57 | maxFiles: 6 58 | otelLogExporter: 59 | enabled: false 60 | otelTraceExporter: 61 | enabled: false 62 | -------------------------------------------------------------------------------- /deploy/helm/ct.yaml: -------------------------------------------------------------------------------- 1 | # This file is used for chart-testing (https://github.com/helm/chart-testing) 2 | # The name "ct.yaml" is not very self-descriptive but it is the default that chart-testing is looking for 3 | --- 4 | remote: origin 5 | target-branch: main 6 | chart-dirs: 7 | - deploy/helm 8 | all: true 9 | -------------------------------------------------------------------------------- /deploy/stackable-operators-ns.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: stackable-operators 6 | -------------------------------------------------------------------------------- /docs/antora.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: home 3 | version: "nightly" 4 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-dags-configmap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | image: 8 | productVersion: 2.10.5 9 | clusterConfig: 10 | loadExamples: false 11 | exposeConfig: false 12 | credentialsSecret: simple-airflow-credentials 13 | volumes: 14 | - name: cm-dag # <3> 15 | configMap: 16 | name: cm-dag # <4> 17 | volumeMounts: 18 | - name: cm-dag # <5> 19 | mountPath: /dags/test_airflow_dag.py # <6> 20 | subPath: test_airflow_dag.py # <7> 21 | webservers: 22 | config: 23 | listenerClass: external-unstable 24 | roleGroups: 25 | default: 26 | envOverrides: 27 | AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8> 28 | replicas: 1 29 | celeryExecutors: 30 | roleGroups: 31 | default: 32 | envOverrides: 33 | AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8> 34 | replicas: 2 35 | schedulers: 36 | roleGroups: 37 | default: 38 | envOverrides: 39 | AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8> 40 | replicas: 1 41 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-gitsync.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | image: 8 | productVersion: "2.10.5" 9 | clusterConfig: 10 | loadExamples: false 11 | exposeConfig: false 12 | credentialsSecret: test-airflow-credentials # <1> 13 | dagsGitSync: # <2> 14 | - repo: https://github.com/stackabletech/airflow-operator # <3> 15 | branch: "main" # <4> 16 | gitFolder: "tests/templates/kuttl/mount-dags-gitsync/dags" # <5> 17 | depth: 10 # <6> 18 | wait: 20s # <7> 19 | credentialsSecret: git-credentials # <8> 20 | gitSyncConf: # <9> 21 | --rev: HEAD # <10> 22 | # --rev: git-sync-tag # N.B. tag must be covered by "depth" (the number of commits to clone) 23 | # --rev: 39ee3598bd9946a1d958a448c9f7d3774d7a8043 # N.B. commit must be covered by "depth" 24 | --git-config: http.sslCAInfo:/tmp/ca-cert/ca.crt # <11> 25 | webservers: 26 | ... 27 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-incluster.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | image: 8 | productVersion: 2.10.5 9 | clusterConfig: 10 | loadExamples: false 11 | exposeConfig: false 12 | credentialsSecret: simple-airflow-credentials 13 | webservers: 14 | config: 15 | listenerClass: external-unstable 16 | roleGroups: 17 | default: 18 | envOverrides: 19 | AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" 20 | replicas: 1 21 | schedulers: 22 | roleGroups: 23 | default: 24 | envOverrides: 25 | AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" 26 | replicas: 1 27 | celeryExecutors: 28 | roleGroups: 29 | default: 30 | envOverrides: 31 | AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" 32 | replicas: 1 33 | # in case of using kubernetesExecutors 34 | # kubernetesExecutors: 35 | # envOverrides: 36 | # AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" 37 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-logging.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: airflow.stackable.tech/v1alpha1 2 | kind: AirflowCluster 3 | metadata: 4 | name: airflow 5 | spec: 6 | image: 7 | productVersion: 2.10.5 8 | clusterConfig: {} 9 | webservers: 10 | config: 11 | listenerClass: external-unstable 12 | envOverrides: &envOverrides 13 | AIRFLOW__LOGGING__REMOTE_LOGGING: "True" 14 | AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3:///airflow-task-logs/ 15 | # The name of the S3 connection created in the Airflow Web UI 16 | AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio 17 | roleGroups: 18 | default: 19 | replicas: 1 20 | schedulers: 21 | envOverrides: *envOverrides 22 | roleGroups: 23 | default: 24 | replicas: 1 25 | kubernetesExecutors: 26 | envOverrides: *envOverrides 27 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-xcom.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: airflow.stackable.tech/v1alpha1 2 | kind: AirflowCluster 3 | metadata: 4 | name: airflow 5 | spec: 6 | image: 7 | productVersion: 2.10.5 8 | clusterConfig: {} 9 | webservers: 10 | config: 11 | listenerClass: external-unstable 12 | envOverrides: &envOverrides 13 | AIRFLOW__CORE__XCOM_BACKEND: airflow.providers.common.io.xcom.backend.XComObjectStorageBackend 14 | # The connection id is obtained from the user part of the url that you will provide 15 | AIRFLOW__COMMON_IO__XCOM_OBJECTSTORAGE_PATH: s3://minio@/airflow-xcom 16 | # Any object smaller than the threshold in bytes will be stored in the database and anything larger will be be put in object storage 17 | AIRFLOW__COMMON_IO__XCOM_OBJECTSTORAGE_THRESHOLD: "1024" # 1KiB 18 | AIRFLOW__COMMON_IO__XCOM_OBJECTSTORAGE_COMPRESSION: gzip 19 | roleGroups: 20 | default: 21 | replicas: 1 22 | schedulers: 23 | envOverrides: *envOverrides 24 | roleGroups: 25 | default: 26 | replicas: 1 27 | kubernetesExecutors: 28 | envOverrides: *envOverrides 29 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-secret.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: simple-airflow-credentials 6 | type: Opaque 7 | stringData: 8 | adminUser.username: airflow 9 | adminUser.firstname: Airflow 10 | adminUser.lastname: Admin 11 | adminUser.email: airflow@airflow.com 12 | adminUser.password: airflow 13 | connections.secretKey: thisISaSECRET_1234 14 | connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 15 | # Only needed when using celery workers (instead of Kubernetes executors) 16 | connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 17 | connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 18 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-spark-clusterrole.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: airflow-spark-clusterrole 6 | rules: 7 | - apiGroups: 8 | - spark.stackable.tech 9 | resources: 10 | - sparkapplications 11 | verbs: 12 | - create 13 | - get 14 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-spark-clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRoleBinding 4 | metadata: 5 | name: airflow-spark-clusterrole-binding 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: ClusterRole 9 | name: airflow-spark-clusterrole 10 | subjects: 11 | - apiGroup: rbac.authorization.k8s.io 12 | kind: Group 13 | name: system:serviceaccounts 14 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-configmap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: cm-dag # <1> 6 | data: 7 | test_airflow_dag.py: | # <2> 8 | from datetime import datetime, timedelta 9 | from airflow import DAG 10 | from airflow.operators.bash import BashOperator 11 | from airflow.operators.dummy import DummyOperator 12 | 13 | with DAG( 14 | dag_id='test_airflow_dag', 15 | schedule_interval='0 0 * * *', 16 | start_date=datetime(2021, 1, 1), 17 | catchup=False, 18 | dagrun_timeout=timedelta(minutes=60), 19 | tags=['example', 'example2'], 20 | params={"example_key": "example_value"}, 21 | ) as dag: 22 | run_this_last = DummyOperator( 23 | task_id='run_this_last', 24 | ) 25 | 26 | # [START howto_operator_bash] 27 | run_this = BashOperator( 28 | task_id='run_after_loop', 29 | bash_command='echo 1', 30 | ) 31 | # [END howto_operator_bash] 32 | 33 | run_this >> run_this_last 34 | 35 | for i in range(3): 36 | task = BashOperator( 37 | task_id='runme_' + str(i), 38 | bash_command='echo "{{ task_instance_key_str }}" && sleep 1', 39 | ) 40 | task >> run_this 41 | 42 | # [START howto_operator_bash_template] 43 | also_run_this = BashOperator( 44 | task_id='also_run_this', 45 | bash_command='echo "run_id={{ run_id }} | dag_run={{ dag_run }}"', 46 | ) 47 | # [END howto_operator_bash_template] 48 | also_run_this >> run_this_last 49 | 50 | # [START howto_operator_bash_skip] 51 | this_will_skip = BashOperator( 52 | task_id='this_will_skip', 53 | bash_command='echo "hello world"; exit 99;', 54 | dag=dag, 55 | ) 56 | # [END howto_operator_bash_skip] 57 | this_will_skip >> run_this_last 58 | 59 | if __name__ == "__main__": 60 | dag.cli() 61 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-pyspark-pi.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: spark.stackable.tech/v1alpha1 3 | kind: SparkApplication 4 | metadata: 5 | name: pyspark-pi 6 | spec: 7 | sparkImage: 8 | productVersion: 3.5.2 9 | mode: cluster 10 | mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py 11 | executor: 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: simple-airflow-credentials 6 | type: Opaque 7 | stringData: 8 | adminUser.username: airflow 9 | adminUser.firstname: Airflow 10 | adminUser.lastname: Admin 11 | adminUser.email: airflow@airflow.com 12 | adminUser.password: airflow 13 | connections.secretKey: thisISaSECRET_1234 14 | connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 15 | # Only needed when using celery workers (instead of Kubernetes executors) 16 | connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 17 | connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 18 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/airflow.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | image: 8 | productVersion: 2.10.5 9 | clusterConfig: 10 | loadExamples: true 11 | exposeConfig: false 12 | credentialsSecret: simple-airflow-credentials 13 | webservers: 14 | config: 15 | listenerClass: external-unstable 16 | roleGroups: 17 | default: 18 | replicas: 1 19 | celeryExecutors: 20 | roleGroups: 21 | default: 22 | replicas: 2 23 | config: 24 | resources: 25 | cpu: 26 | min: 400m 27 | max: 800m 28 | memory: 29 | limit: 2Gi 30 | schedulers: 31 | roleGroups: 32 | default: 33 | replicas: 1 34 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/install_output.txt: -------------------------------------------------------------------------------- 1 | Installed commons=0.0.0-dev operator 2 | Installed secret=0.0.0-dev operator 3 | Installed listener=0.0.0-dev operator 4 | Installed airflow=0.0.0-dev operator 5 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/install_output.txt.j2: -------------------------------------------------------------------------------- 1 | Installed commons={{ versions.commons }} operator 2 | Installed secret={{ versions.secret }} operator 3 | Installed listener={{ versions.listener }} operator 4 | Installed airflow={{ versions.airflow }} operator 5 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/test_getting_started_helm.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "$0")" 5 | ./getting_started.sh helm 6 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/test_getting_started_stackablectl.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "$0")" 5 | ./getting_started.sh stackablectl 6 | -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_connection_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_connection_ui.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_dag_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_dag_graph.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_dag_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_dag_log.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_dag_log_opensearch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_dag_log_opensearch.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_dag_s3_logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_dag_s3_logs.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_dags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_dags.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_edit_s3_connection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_edit_s3_connection.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_login.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_running.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_running.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_security.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_security.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_security_ldap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/airflow_security_ldap.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/getting_started/airflow_dags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/getting_started/airflow_dags.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/getting_started/airflow_login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/getting_started/airflow_login.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/getting_started/airflow_pods.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/getting_started/airflow_pods.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/getting_started/airflow_running.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/docs/modules/airflow/images/getting_started/airflow_running.png -------------------------------------------------------------------------------- /docs/modules/airflow/pages/getting_started/index.adoc: -------------------------------------------------------------------------------- 1 | = Getting started 2 | :description: Get started with the Stackable Operator for Apache Airflow by installing the operator, SQL database, and Redis, then setting up and running your first DAG. 3 | 4 | This guide gets you started with Airflow using the Stackable Operator. 5 | It guides you through the installation of the Operator as well as an SQL database and Redis instance for trial usage, setting up your first Airflow cluster and connecting to it, and viewing and running one of the example workflows (called DAGs = Direct Acyclic Graphs). 6 | 7 | == Prerequisites for this guide 8 | 9 | You need: 10 | 11 | * a Kubernetes cluster 12 | * kubectl 13 | * Helm 14 | 15 | Resource sizing depends on cluster type(s), usage and scope, but as a minimum starting point the following resources are recommended for this operator: 16 | 17 | include::partial$hardware-requirements.adoc[] 18 | 19 | == What's next 20 | 21 | The Guide is divided into two steps: 22 | 23 | * xref:getting_started/installation.adoc[Installing the Operators]. 24 | * xref:getting_started/first_steps.adoc[Setting up the Airflow cluster and running an example DAG]. 25 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/reference/commandline-parameters.adoc: -------------------------------------------------------------------------------- 1 | = Command Line Parameters 2 | 3 | This operator accepts the following command line parameters: 4 | 5 | == product-config 6 | 7 | *Default value*: `/etc/stackable/airflow-operator/config-spec/properties.yaml` 8 | 9 | *Required*: false 10 | 11 | *Multiple values:* false 12 | 13 | [source] 14 | ---- 15 | stackable-airflow-operator run --product-config /foo/bar/properties.yaml 16 | ---- 17 | 18 | == watch-namespace 19 | 20 | *Default value*: All namespaces 21 | 22 | *Required*: false 23 | 24 | *Multiple values:* false 25 | 26 | The operator **only** watches for resources in the provided namespace `test`: 27 | 28 | [source] 29 | ---- 30 | stackable-airflow-operator run --watch-namespace test 31 | ---- 32 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/reference/crds.adoc: -------------------------------------------------------------------------------- 1 | = CRD Reference 2 | 3 | Find all CRD reference for the Stackable Operator for Apache Airfow at: {crd-docs-base-url}/airflow-operator/{crd-docs-version}. 4 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/reference/index.adoc: -------------------------------------------------------------------------------- 1 | = Reference 2 | 3 | Consult the reference documentation section to find exhaustive information on: 4 | 5 | * Descriptions and default values of all properties in the CRDs used by this operator in the xref:reference/crds.adoc[]. 6 | * The xref:reference/commandline-parameters.adoc[] and xref:reference/environment-variables.adoc[] accepted by the operator. 7 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/required-external-components.adoc: -------------------------------------------------------------------------------- 1 | = Required external components 2 | :description: Airflow requires PostgreSQL or MySQL for database support, and Redis for Celery executors. 3 | :airflow-prerequisites: https://airflow.apache.org/docs/apache-airflow/stable/installation/prerequisites.html 4 | 5 | Airflow requires an SQL database to operate. 6 | The {airflow-prerequisites}[Airflow documentation] specifies: 7 | 8 | Fully supported for production usage: 9 | 10 | * PostgreSQL: 12, 13, 14, 15, 16 11 | * MySQL: 8.0, Innovation 12 | 13 | The Celery exectutor also requires: 14 | 15 | * Redis (any stable version) 16 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/index.adoc: -------------------------------------------------------------------------------- 1 | = Usage guide 2 | :description: Practical instructions to make the most out of the Stackable operator for Apache Airflow. 3 | 4 | Practical instructions to make the most out of the Stackable operator for Apache Airflow. 5 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/listenerclass.adoc: -------------------------------------------------------------------------------- 1 | = Service exposition with ListenerClasses 2 | :description: Configure Airflow service exposure with ListenerClasses: cluster-internal, external-unstable, or external-stable. 3 | 4 | The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Webserver pod. 5 | The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.webservers.config.listenerClass`: 6 | 7 | [source,yaml] 8 | ---- 9 | spec: 10 | webservers: 11 | config: 12 | listenerClass: external-unstable # <1> 13 | schedulers: 14 | ... 15 | celeryExecutors: 16 | ... 17 | ---- 18 | <1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`). 19 | This can be set only for the webservers role. 20 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/logging.adoc: -------------------------------------------------------------------------------- 1 | = Log aggregation 2 | :description: Forward Airflow logs to a Vector aggregator by configuring the ConfigMap and enabling the log agent. 3 | 4 | The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent: 5 | 6 | [source,yaml] 7 | ---- 8 | spec: 9 | clusterConfig: 10 | vectorAggregatorConfigMapName: vector-aggregator-discovery 11 | webservers: 12 | config: 13 | logging: 14 | enableVectorAgent: true 15 | containers: 16 | airflow: 17 | loggers: 18 | "flask_appbuilder": 19 | level: WARN 20 | celeryExecutors: 21 | config: 22 | logging: 23 | enableVectorAgent: true 24 | containers: 25 | airflow: 26 | loggers: 27 | "airflow.processor": 28 | level: INFO 29 | schedulers: 30 | config: 31 | logging: 32 | enableVectorAgent: true 33 | containers: 34 | airflow: 35 | loggers: 36 | "airflow.processor_manager": 37 | level: INFO 38 | ---- 39 | 40 | Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. 41 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/monitoring.adoc: -------------------------------------------------------------------------------- 1 | = Monitoring 2 | :description: Airflow instances export Prometheus metrics for monitoring. 3 | 4 | The managed Airflow instances are automatically configured to export Prometheus metrics. 5 | See xref:operators:monitoring.adoc[] for more details. 6 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/operations/cluster-operations.adoc: -------------------------------------------------------------------------------- 1 | = Cluster Operation 2 | 3 | Airflow installations can be configured with different cluster operations like pausing reconciliation or stopping the cluster. See xref:concepts:operations/cluster_operations.adoc[cluster operations] for more details. 4 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/operations/index.adoc: -------------------------------------------------------------------------------- 1 | = Operations 2 | 3 | This section of the documentation is intended for the operations teams that maintain a Stackable Data Platform installation. 4 | 5 | Read the xref:concepts:operations/index.adoc[concepts page on operations] that contains the necessary details to operate the platform in a production environment. 6 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/operations/pod-disruptions.adoc: -------------------------------------------------------------------------------- 1 | = Allowed Pod disruptions 2 | 3 | You can configure the permitted Pod disruptions for Airflow nodes as described in xref:concepts:operations/pod_disruptions.adoc[]. 4 | 5 | Unless you configure something else or disable the default PodDisruptionBudgets (PDBs), the operator writes the following PDBs: 6 | 7 | == Schedulers 8 | Allow only a single scheduler to be offline at any given time, regardless of the number of replicas or `roleGroups`. 9 | 10 | == Webservers 11 | Allow only a single webserver to be offline at any given time, regardless of the number of replicas or `roleGroups`. 12 | 13 | == Executors 14 | * In the case of Celery executors, allow only a single executor to be offline at any given time, regardless of the number of replicas or `roleGroups`. 15 | * In the case of Kubernetes executors, don't deploy any PDB, as it's Airflows responsibility to take care of the executor Pods. 16 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/operations/pod-placement.adoc: -------------------------------------------------------------------------------- 1 | = Pod placement 2 | 3 | You can configure the Pod placement of the Airflow pods as described in xref:concepts:operations/pod_placement.adoc[]. 4 | 5 | The default affinities created by the operator are: 6 | 7 | 1. Co-locate all the Airflow Pods (weight 20) 8 | 2. Distribute all Pods within the same role (worker, webserver, scheduler) (weight 70) 9 | 10 | == Kubernetes executors 11 | 12 | The Kubernetes executors are no different, they try to co-locate with other Airflow Pods and (with a higher priority) distribute all executors. 13 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/storage-resources.adoc: -------------------------------------------------------------------------------- 1 | = Resource Requests 2 | :description: Find out about minimal HA Airflow requirements for CPU and memory, with defaults for schedulers, Celery executors, webservers using Kubernetes resource limits. 3 | 4 | include::home:concepts:stackable_resource_requests.adoc[] 5 | 6 | A minimal HA setup consisting of 2 schedulers, 2 workers and 2 webservers has the following https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/[resource requirements]: 7 | 8 | * `5700m` CPU request 9 | * `17400m` CPU limit 10 | * `10752Mi` memory request and limit 11 | 12 | Corresponding to the values above, the operator uses the following resource defaults: 13 | 14 | [source,yaml] 15 | ---- 16 | spec: 17 | schedulers: 18 | config: 19 | resources: 20 | cpu: 21 | min: 500m 22 | max: "2" 23 | memory: 24 | limit: 512Mi 25 | celeryExecutors: 26 | config: 27 | resources: 28 | cpu: 29 | min: 500m 30 | max: "2" 31 | memory: 32 | limit: 2Gi 33 | webservers: 34 | config: 35 | resources: 36 | cpu: 37 | min: 500m 38 | max: "2" 39 | memory: 40 | limit: 2Gi 41 | ---- 42 | -------------------------------------------------------------------------------- /docs/modules/airflow/partials/hardware-requirements.adoc: -------------------------------------------------------------------------------- 1 | * 0.2 cores (e.g. i5 or similar) 2 | * 256MB RAM 3 | -------------------------------------------------------------------------------- /docs/modules/airflow/partials/nav.adoc: -------------------------------------------------------------------------------- 1 | * xref:airflow:getting_started/index.adoc[] 2 | ** xref:airflow:getting_started/installation.adoc[] 3 | ** xref:airflow:getting_started/first_steps.adoc[] 4 | * xref:airflow:required-external-components.adoc[] 5 | * xref:airflow:usage-guide/index.adoc[] 6 | ** xref:airflow:usage-guide/mounting-dags.adoc[] 7 | ** xref:airflow:usage-guide/applying-custom-resources.adoc[] 8 | ** xref:airflow:usage-guide/listenerclass.adoc[] 9 | ** xref:airflow:usage-guide/storage-resources.adoc[] 10 | ** xref:airflow:usage-guide/security.adoc[] 11 | ** xref:airflow:usage-guide/logging.adoc[] 12 | ** xref:airflow:usage-guide/monitoring.adoc[] 13 | ** xref:airflow:usage-guide/using-kubernetes-executors.adoc[] 14 | ** xref:airflow:usage-guide/overrides.adoc[] 15 | ** xref:airflow:usage-guide/operations/index.adoc[] 16 | *** xref:airflow:usage-guide/operations/cluster-operations.adoc[] 17 | *** xref:airflow:usage-guide/operations/pod-placement.adoc[] 18 | *** xref:airflow:usage-guide/operations/pod-disruptions.adoc[] 19 | *** xref:airflow:usage-guide/operations/graceful-shutdown.adoc[] 20 | * xref:airflow:reference/index.adoc[] 21 | ** xref:airflow:reference/crds.adoc[] 22 | *** {crd-docs}/airflow.stackable.tech/airflowcluster/v1alpha1/[AirflowCluster {external-link-icon}^] 23 | ** xref:airflow:reference/commandline-parameters.adoc[] 24 | ** xref:airflow:reference/environment-variables.adoc[] 25 | -------------------------------------------------------------------------------- /docs/modules/airflow/partials/supported-versions.adoc: -------------------------------------------------------------------------------- 1 | // The version ranges supported by Airflow-Operator 2 | // This is a separate file, since it is used by both the direct Airflow-Operator documentation, and the overarching 3 | // Stackable Platform documentation. 4 | 5 | - 2.10.5 6 | - 2.10.4 (deprecated) 7 | - 2.9.3 (LTS) 8 | -------------------------------------------------------------------------------- /docs/templating_vars.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | helm: 3 | repo_name: sdp-charts 4 | repo_url: oci.stackable.tech 5 | versions: 6 | commons: 0.0.0-dev 7 | secret: 0.0.0-dev 8 | listener: 0.0.0-dev 9 | airflow: 0.0.0-dev 10 | postgresql: 16.5.0 11 | redis: 20.11.3 12 | -------------------------------------------------------------------------------- /examples/simple-airflow-cluster.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: simple-airflow-credentials 6 | type: Opaque 7 | stringData: 8 | adminUser.username: airflow 9 | adminUser.firstname: Airflow 10 | adminUser.lastname: Admin 11 | adminUser.email: airflow@airflow.com 12 | adminUser.password: airflow 13 | connections.secretKey: thisISaSECRET_1234 14 | connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 15 | # Only needed when using celery workers (instead of Kubernetes executors) 16 | connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 17 | connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 18 | --- 19 | apiVersion: airflow.stackable.tech/v1alpha1 20 | kind: AirflowCluster 21 | metadata: 22 | name: airflow 23 | spec: 24 | image: 25 | productVersion: 2.10.5 26 | clusterConfig: 27 | loadExamples: true 28 | exposeConfig: false 29 | credentialsSecret: simple-airflow-credentials 30 | webservers: 31 | config: 32 | listenerClass: external-unstable 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | celeryExecutors: 37 | roleGroups: 38 | default: 39 | replicas: 2 40 | schedulers: 41 | roleGroups: 42 | default: 43 | replicas: 1 44 | -------------------------------------------------------------------------------- /nix/README.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | # Updating nix dependencies 7 | 8 | ## Run the following for an operator 9 | 10 | > [!NOTE] 11 | > We track the `master` branch of crate2nix as that is relatively up to date, but the releases are infrequent. 12 | 13 | ```shell 14 | niv update crate2nix 15 | niv update nixpkgs 16 | niv update beku.py -b X.Y.Z # Using the release tag 17 | ``` 18 | 19 | ### Test 20 | 21 | - Run make `regenerate-nix` to ensure crate2nix works 22 | - Run a smoke test to ensure beku.py works. 23 | - Run `make run-dev` to ensure nixpkgs are fine. 24 | 25 | ## Update operator-templating 26 | 27 | Do the same as above, but from `template/` 28 | -------------------------------------------------------------------------------- /nix/meta.json: -------------------------------------------------------------------------------- 1 | {"operator": {"name": "airflow-operator", "pretty_string": "Apache Airflow", "product_string": "airflow", "url": "stackabletech/airflow-operator.git"}} 2 | -------------------------------------------------------------------------------- /nix/sources.json: -------------------------------------------------------------------------------- 1 | { 2 | "beku.py": { 3 | "branch": "0.0.10", 4 | "description": "Test suite expander for Stackable Kuttl tests.", 5 | "homepage": null, 6 | "owner": "stackabletech", 7 | "repo": "beku.py", 8 | "rev": "fc75202a38529a4ac6776dd8a5dfee278d927f58", 9 | "sha256": "152yary0p11h87yabv74jnwkghsal7lx16az0qlzrzdrs6n5v8id", 10 | "type": "tarball", 11 | "url": "https://github.com/stackabletech/beku.py/archive/fc75202a38529a4ac6776dd8a5dfee278d927f58.tar.gz", 12 | "url_template": "https://github.com///archive/.tar.gz" 13 | }, 14 | "crate2nix": { 15 | "branch": "master", 16 | "description": "nix build file generator for rust crates", 17 | "homepage": "", 18 | "owner": "kolloch", 19 | "repo": "crate2nix", 20 | "rev": "be31feae9a82c225c0fd1bdf978565dc452a483a", 21 | "sha256": "14d0ymlrwk7dynv35qcw4xn0dylfpwjmf6f8znflbk2l6fk23l12", 22 | "type": "tarball", 23 | "url": "https://github.com/kolloch/crate2nix/archive/be31feae9a82c225c0fd1bdf978565dc452a483a.tar.gz", 24 | "url_template": "https://github.com///archive/.tar.gz" 25 | }, 26 | "nixpkgs": { 27 | "branch": "nixpkgs-unstable", 28 | "description": "Nix Packages collection", 29 | "homepage": "", 30 | "owner": "NixOS", 31 | "repo": "nixpkgs", 32 | "rev": "b1bebd0fe266bbd1820019612ead889e96a8fa2d", 33 | "sha256": "0fl2dji5whjydbxby9b7kqyqx9m4k44p72x1q28kfnx5m67nyqij", 34 | "type": "tarball", 35 | "url": "https://github.com/NixOS/nixpkgs/archive/b1bebd0fe266bbd1820019612ead889e96a8fa2d.tar.gz", 36 | "url_template": "https://github.com///archive/.tar.gz" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "local>stackabletech/.github:renovate-config" 5 | ], 6 | "ignorePaths": [".github/workflows/build.yml", ".github/workflows/general_daily_security.yml", ".github/workflows/integration-test.yml", ".github/workflows/pr_pre-commit.yaml"] 7 | } 8 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | # DO NOT EDIT, this file is generated by operator-templating 2 | [toolchain] 3 | channel = "1.85.0" 4 | profile = "default" 5 | -------------------------------------------------------------------------------- /rust/operator-binary/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "stackable-airflow-operator" 3 | description = "Stackable Operator for Apache Airflow" 4 | version.workspace = true 5 | authors.workspace = true 6 | license.workspace = true 7 | edition.workspace = true 8 | repository.workspace = true 9 | publish = false 10 | 11 | [dependencies] 12 | product-config.workspace = true 13 | stackable-operator.workspace = true 14 | 15 | anyhow.workspace = true 16 | clap.workspace = true 17 | const_format.workspace = true 18 | fnv.workspace = true 19 | futures.workspace = true 20 | serde.workspace = true 21 | serde_json.workspace = true 22 | serde_yaml.workspace = true 23 | snafu.workspace = true 24 | strum.workspace = true 25 | tokio.workspace = true 26 | tracing.workspace = true 27 | indoc.workspace = true 28 | 29 | [build-dependencies] 30 | built.workspace = true 31 | 32 | [dev-dependencies] 33 | rstest.workspace = true 34 | serde_yaml.workspace = true 35 | -------------------------------------------------------------------------------- /rust/operator-binary/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | built::write_built_file().unwrap(); 3 | } 4 | -------------------------------------------------------------------------------- /rust/operator-binary/src/crd/authorization.rs: -------------------------------------------------------------------------------- 1 | use stackable_operator::{client::Client, commons::opa::OpaApiVersion, time::Duration}; 2 | 3 | use crate::crd::{AirflowAuthorization, AirflowOpaConfig, v1alpha1}; 4 | 5 | pub struct AirflowAuthorizationResolved { 6 | pub opa: Option, 7 | } 8 | 9 | impl AirflowAuthorizationResolved { 10 | pub async fn from_authorization_config( 11 | client: &Client, 12 | airflow: &v1alpha1::AirflowCluster, 13 | authorization: &Option, 14 | ) -> Result { 15 | let opa = if let Some(AirflowAuthorization { 16 | opa: Some(opa_config), 17 | }) = authorization 18 | { 19 | Some(OpaConfigResolved::from_opa_config(client, airflow, opa_config).await?) 20 | } else { 21 | None 22 | }; 23 | Ok(AirflowAuthorizationResolved { opa }) 24 | } 25 | } 26 | 27 | pub struct OpaConfigResolved { 28 | pub connection_string: String, 29 | pub cache_entry_time_to_live: Duration, 30 | pub cache_max_entries: u32, 31 | } 32 | 33 | impl OpaConfigResolved { 34 | pub async fn from_opa_config( 35 | client: &Client, 36 | airflow: &v1alpha1::AirflowCluster, 37 | airflow_opa_config: &AirflowOpaConfig, 38 | ) -> Result { 39 | let connection_string = airflow_opa_config 40 | .opa 41 | .full_document_url_from_config_map(client, airflow, None, OpaApiVersion::V1) 42 | .await?; 43 | Ok(OpaConfigResolved { 44 | connection_string, 45 | cache_entry_time_to_live: airflow_opa_config.cache.entry_time_to_live, 46 | cache_max_entries: airflow_opa_config.cache.max_entries, 47 | }) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /rust/operator-binary/src/operations/graceful_shutdown.rs: -------------------------------------------------------------------------------- 1 | use snafu::{ResultExt, Snafu}; 2 | use stackable_operator::builder::pod::PodBuilder; 3 | 4 | use crate::crd::{AirflowConfig, ExecutorConfig}; 5 | 6 | #[derive(Debug, Snafu)] 7 | pub enum Error { 8 | #[snafu(display("Failed to set terminationGracePeriod"))] 9 | SetTerminationGracePeriod { 10 | source: stackable_operator::builder::pod::Error, 11 | }, 12 | } 13 | 14 | pub fn add_airflow_graceful_shutdown_config( 15 | merged_config: &AirflowConfig, 16 | pod_builder: &mut PodBuilder, 17 | ) -> Result<(), Error> { 18 | // This must be always set by the merge mechanism, as we provide a default value, 19 | // users can not disable graceful shutdown. 20 | if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout { 21 | pod_builder 22 | .termination_grace_period(&graceful_shutdown_timeout) 23 | .context(SetTerminationGracePeriodSnafu)?; 24 | } 25 | 26 | Ok(()) 27 | } 28 | 29 | pub fn add_executor_graceful_shutdown_config( 30 | merged_config: &ExecutorConfig, 31 | pod_builder: &mut PodBuilder, 32 | ) -> Result<(), Error> { 33 | // This must be always set by the merge mechanism, as we provide a default value, 34 | // users can not disable graceful shutdown. 35 | if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout { 36 | pod_builder 37 | .termination_grace_period(&graceful_shutdown_timeout) 38 | .context(SetTerminationGracePeriodSnafu)?; 39 | } 40 | 41 | Ok(()) 42 | } 43 | -------------------------------------------------------------------------------- /rust/operator-binary/src/operations/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod graceful_shutdown; 2 | pub mod pdb; 3 | -------------------------------------------------------------------------------- /rust/operator-binary/src/util.rs: -------------------------------------------------------------------------------- 1 | use stackable_operator::k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector}; 2 | 3 | pub fn env_var_from_secret(var_name: &str, secret: &str, secret_key: &str) -> EnvVar { 4 | EnvVar { 5 | name: String::from(var_name), 6 | value_from: Some(EnvVarSource { 7 | secret_key_ref: Some(SecretKeySelector { 8 | name: String::from(secret), 9 | key: String::from(secret_key), 10 | ..Default::default() 11 | }), 12 | ..Default::default() 13 | }), 14 | ..Default::default() 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # This file includes unstable features, so you need to run "cargo +nightly fmt" to format your code. 2 | # It's also ok to use the stable toolchain by simple running "cargo fmt", but using the nigthly formatter is prefered. 3 | 4 | # https://doc.rust-lang.org/nightly/edition-guide/rust-2024/rustfmt-style-edition.html 5 | style_edition = "2024" 6 | imports_granularity = "Crate" 7 | group_imports = "StdExternalCrate" 8 | reorder_impl_items = true 9 | use_field_init_shorthand = true 10 | -------------------------------------------------------------------------------- /scripts/docs_templating.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Reads a file with variables to insert into templates, and templates all .*.j2 files 5 | # in the 'docs' directory. 6 | # 7 | # dependencies 8 | # pip install jinja2-cli 9 | 10 | docs_dir="$(dirname "$0")/../docs" 11 | templating_vars_file="$docs_dir/templating_vars.yaml" 12 | 13 | # Check if files need templating 14 | if [[ -z $(find "$docs_dir" -name '*.j2') ]]; 15 | then 16 | echo "No files need templating, exiting." 17 | exit 18 | fi 19 | 20 | # Check if jinja2 is there 21 | if ! command -v jinja2 &> /dev/null 22 | then 23 | echo "jinja2 could not be found. Use 'pip install jinja2-cli' to install it." 24 | exit 1 25 | fi 26 | 27 | # Check if templating vars file exists 28 | if [[ ! -f "$templating_vars_file" ]]; 29 | then 30 | echo "$templating_vars_file does not exist, cannot start templating." 31 | fi 32 | 33 | find "$docs_dir" -name '*.j2' | 34 | while read -r file 35 | do 36 | new_file_name=${file%.j2} # Remove .j2 suffix 37 | echo "templating $new_file_name" 38 | jinja2 "$file" "$templating_vars_file" -o "$new_file_name" 39 | done 40 | 41 | echo "done" 42 | -------------------------------------------------------------------------------- /scripts/generate-manifests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This script reads a Helm chart from deploy/helm/airflow-operator and 3 | # generates manifest files into deploy/manifestss 4 | set -e 5 | 6 | tmp=$(mktemp -d ./manifests-XXXXX) 7 | 8 | helm template --output-dir "$tmp" \ 9 | --include-crds \ 10 | --name-template airflow-operator \ 11 | deploy/helm/airflow-operator 12 | 13 | for file in "$tmp"/airflow-operator/*/*; do 14 | yq eval -i 'del(.. | select(has("app.kubernetes.io/managed-by")) | ."app.kubernetes.io/managed-by")' /dev/stdin < "$file" 15 | yq eval -i 'del(.. | select(has("helm.sh/chart")) | ."helm.sh/chart")' /dev/stdin < "$file" 16 | sed -i '/# Source: .*/d' "$file" 17 | done 18 | 19 | cp -r "$tmp"/airflow-operator/*/* deploy/manifests/ 20 | 21 | rm -rf "$tmp" 22 | -------------------------------------------------------------------------------- /scripts/render_readme.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Check if jinja2 is there 5 | if ! command -v jinja2 &> /dev/null 6 | then 7 | echo "jinja2 could not be found. Use 'pip install jinja2-cli' to install it." 8 | exit 1 9 | fi 10 | 11 | SCRIPT_DIR=$(dirname "$0") 12 | cd "$SCRIPT_DIR/../.readme" 13 | jinja2 README.md.j2 -o ../README.md 14 | cd .. 15 | 16 | python3 scripts/ensure_one_trailing_newline.py README.md 17 | -------------------------------------------------------------------------------- /scripts/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ./scripts/run-tests "$@" 4 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | let 2 | self = import ./. {}; 3 | inherit (self) sources pkgs meta; 4 | 5 | beku = pkgs.callPackage (sources."beku.py" + "/beku.nix") {}; 6 | cargoDependencySetOfCrate = crate: [ crate ] ++ pkgs.lib.concatMap cargoDependencySetOfCrate (crate.dependencies ++ crate.buildDependencies); 7 | cargoDependencySet = pkgs.lib.unique (pkgs.lib.flatten (pkgs.lib.mapAttrsToList (crateName: crate: cargoDependencySetOfCrate crate.build) self.cargo.workspaceMembers)); 8 | in pkgs.mkShell rec { 9 | name = meta.operator.name; 10 | 11 | packages = with pkgs; [ 12 | ## cargo et-al 13 | rustup # this breaks pkg-config if it is in the nativeBuildInputs 14 | cargo-udeps 15 | 16 | ## Extra dependencies for use in a pure env (nix-shell --pure) 17 | ## These are mosuly useful for maintainers of this shell.nix 18 | ## to ensure all the dependencies are caught. 19 | # cacert 20 | # vim nvim nano 21 | ]; 22 | 23 | # derivation runtime dependencies 24 | buildInputs = pkgs.lib.concatMap (crate: crate.buildInputs) cargoDependencySet; 25 | 26 | # build time dependencies 27 | nativeBuildInputs = pkgs.lib.concatMap (crate: crate.nativeBuildInputs) cargoDependencySet ++ (with pkgs; [ 28 | beku 29 | docker 30 | gettext # for the proper envsubst 31 | git 32 | jq 33 | kind 34 | kubectl 35 | kubernetes-helm 36 | kuttl 37 | nix # this is implied, but needed in the pure env 38 | # tilt already defined in default.nix 39 | which 40 | yq-go 41 | ]); 42 | 43 | LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; 44 | BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include"; 45 | } 46 | -------------------------------------------------------------------------------- /tests/infrastructure.yaml: -------------------------------------------------------------------------------- 1 | instance-size: medium 2 | -------------------------------------------------------------------------------- /tests/kuttl-test.yaml.jinja2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestSuite 4 | testDirs: 5 | {% for testcase in testinput.tests %} 6 | - ./tests/{{ testcase.name }} 7 | {% endfor %} 8 | 9 | startKIND: false 10 | suppress: ["events"] 11 | parallel: 2 12 | 13 | # The timeout (in seconds) is used when namespaces are created or 14 | # deleted, and, if not overridden, in TestSteps, TestAsserts, and 15 | # Commands. If not set, the timeout is 30 seconds by default. 16 | # 17 | # The deletion of a namespace can take a while until all resources, 18 | # especially PersistentVolumeClaims, are gracefully shut down. If the 19 | # timeout is reached in the meantime, even a successful test case is 20 | # considered a failure. 21 | # 22 | # For instance, the termination grace period of the Vector aggregator in 23 | # the logging tests is set to 60 seconds. If there are logs entries 24 | # which could not be forwarded yet to the external aggregator defined in 25 | # the VECTOR_AGGREGATOR environment variable, then the test aggregator 26 | # uses this period of time by trying to forward the events. In this 27 | # case, deleting a namespace with several Pods takes about 90 seconds. 28 | timeout: 300 29 | -------------------------------------------------------------------------------- /tests/release.yaml: -------------------------------------------------------------------------------- 1 | # Contains all operators required to run the test suite. 2 | --- 3 | releases: 4 | # Do not change the name of the release as it's referenced from run_tests.sh 5 | tests: 6 | releaseDate: 1970-01-01 7 | description: Integration test 8 | products: 9 | commons: 10 | operatorVersion: 0.0.0-dev 11 | secret: 12 | operatorVersion: 0.0.0-dev 13 | listener: 14 | operatorVersion: 0.0.0-dev 15 | airflow: 16 | operatorVersion: 0.0.0-dev 17 | opa: 18 | operatorVersion: 0.0.0-dev 19 | spark-k8s: 20 | operatorVersion: 0.0.0-dev 21 | -------------------------------------------------------------------------------- /tests/templates/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/d24fb5dd450abc2ecfd6f15deea83b30aac8562e/tests/templates/.gitkeep -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/02-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/04-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-redis 6 | timeout: 360 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-redis-master 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-redis-replicas 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/04-install-redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-redis 7 | --namespace $NAMESPACE 8 | --version 17.11.3 9 | -f helm-bitnami-redis-values.yaml 10 | --repo https://charts.bitnami.com/bitnami redis 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/06-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/06-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/08-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-install-airflow 6 | timeout: 1200 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow --timeout 301s 9 | --- 10 | apiVersion: apps/v1 11 | kind: StatefulSet 12 | metadata: 13 | name: airflow-webserver-default 14 | status: 15 | readyReplicas: 1 16 | replicas: 1 17 | --- 18 | apiVersion: apps/v1 19 | kind: StatefulSet 20 | metadata: 21 | name: airflow-worker-default 22 | status: 23 | readyReplicas: 2 24 | replicas: 2 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: kuttl.dev/v1beta1 2 | kind: TestStep 3 | metadata: 4 | name: install-airflow 5 | timeout: 480 6 | --- 7 | apiVersion: v1 8 | kind: Secret 9 | metadata: 10 | name: test-airflow-credentials 11 | type: Opaque 12 | stringData: 13 | adminUser.username: airflow 14 | adminUser.firstname: Airflow 15 | adminUser.lastname: Admin 16 | adminUser.email: airflow@airflow.com 17 | adminUser.password: airflow 18 | connections.secretKey: thisISaSECRET_1234 19 | connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow 20 | connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow 21 | connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 22 | --- 23 | apiVersion: airflow.stackable.tech/v1alpha1 24 | kind: AirflowCluster 25 | metadata: 26 | name: airflow 27 | spec: 28 | image: 29 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 30 | custom: "{{ test_scenario['values']['airflow-latest'].split(',')[1] }}" 31 | productVersion: "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 32 | {% else %} 33 | productVersion: "{{ test_scenario['values']['airflow-latest'] }}" 34 | {% endif %} 35 | pullPolicy: IfNotPresent 36 | clusterConfig: 37 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 38 | vectorAggregatorConfigMapName: vector-aggregator-discovery 39 | {% endif %} 40 | credentialsSecret: test-airflow-credentials 41 | webservers: 42 | config: 43 | listenerClass: external-unstable 44 | logging: 45 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 46 | roleGroups: 47 | default: 48 | replicas: 1 49 | celeryExecutors: 50 | config: 51 | logging: 52 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 53 | roleGroups: 54 | default: 55 | replicas: 2 56 | schedulers: 57 | config: 58 | logging: 59 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 60 | roleGroups: 61 | default: 62 | replicas: 1 63 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-pause-airflow 6 | timeout: 180 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=reconciliationPaused airflowclusters.airflow.stackable.tech/airflow --timeout 301s 9 | --- 10 | apiVersion: apps/v1 11 | kind: StatefulSet 12 | metadata: 13 | name: airflow-webserver-default 14 | status: 15 | readyReplicas: 1 16 | replicas: 1 17 | --- 18 | apiVersion: apps/v1 19 | kind: StatefulSet 20 | metadata: 21 | name: airflow-worker-default 22 | status: 23 | readyReplicas: 2 24 | replicas: 2 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/10-pause-airflow.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: kuttl.dev/v1beta1 2 | kind: TestStep 3 | metadata: 4 | name: pause-airflow 5 | timeout: 480 6 | --- 7 | apiVersion: airflow.stackable.tech/v1alpha1 8 | kind: AirflowCluster 9 | metadata: 10 | name: airflow 11 | spec: 12 | clusterOperation: 13 | reconciliationPaused: true 14 | stopped: false 15 | image: 16 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 17 | custom: "{{ test_scenario['values']['airflow-latest'].split(',')[1] }}" 18 | productVersion: "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 19 | {% else %} 20 | productVersion: "{{ test_scenario['values']['airflow-latest'] }}" 21 | {% endif %} 22 | pullPolicy: IfNotPresent 23 | clusterConfig: 24 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 25 | vectorAggregatorConfigMapName: vector-aggregator-discovery 26 | {% endif %} 27 | credentialsSecret: test-airflow-credentials 28 | webservers: 29 | config: 30 | listenerClass: external-unstable 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | celeryExecutors: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 3 # ignored because paused 43 | schedulers: 44 | config: 45 | logging: 46 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 47 | roleGroups: 48 | default: 49 | replicas: 1 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-stop-airflow 6 | timeout: 180 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=stopped airflowclusters.airflow.stackable.tech/airflow --timeout 301s 9 | --- 10 | apiVersion: apps/v1 11 | kind: StatefulSet 12 | metadata: 13 | name: airflow-webserver-default 14 | status: 15 | replicas: 0 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | replicas: 0 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: airflow-scheduler-default 28 | status: 29 | replicas: 0 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/20-stop-airflow.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: kuttl.dev/v1beta1 2 | kind: TestStep 3 | metadata: 4 | name: stop-airflow 5 | timeout: 180 6 | --- 7 | apiVersion: airflow.stackable.tech/v1alpha1 8 | kind: AirflowCluster 9 | metadata: 10 | name: airflow 11 | spec: 12 | clusterOperation: 13 | reconciliationPaused: false 14 | stopped: true 15 | image: 16 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 17 | custom: "{{ test_scenario['values']['airflow-latest'].split(',')[1] }}" 18 | productVersion: "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 19 | {% else %} 20 | productVersion: "{{ test_scenario['values']['airflow-latest'] }}" 21 | {% endif %} 22 | pullPolicy: IfNotPresent 23 | clusterConfig: 24 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 25 | vectorAggregatorConfigMapName: vector-aggregator-discovery 26 | {% endif %} 27 | credentialsSecret: test-airflow-credentials 28 | webservers: 29 | config: 30 | listenerClass: external-unstable 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | celeryExecutors: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 3 # ignored because paused 43 | schedulers: 44 | config: 45 | logging: 46 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 47 | roleGroups: 48 | default: 49 | replicas: 1 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-restart-airflow 6 | timeout: 600 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow --timeout 301s 9 | --- 10 | apiVersion: apps/v1 11 | kind: StatefulSet 12 | metadata: 13 | name: airflow-webserver-default 14 | status: 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | replicas: 3 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: airflow-scheduler-default 28 | status: 29 | replicas: 1 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/30-restart-airflow.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: kuttl.dev/v1beta1 2 | kind: TestStep 3 | metadata: 4 | name: restart-airflow 5 | timeout: 180 6 | --- 7 | apiVersion: airflow.stackable.tech/v1alpha1 8 | kind: AirflowCluster 9 | metadata: 10 | name: airflow 11 | spec: 12 | clusterOperation: 13 | reconciliationPaused: false 14 | stopped: false 15 | image: 16 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 17 | custom: "{{ test_scenario['values']['airflow-latest'].split(',')[1] }}" 18 | productVersion: "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 19 | {% else %} 20 | productVersion: "{{ test_scenario['values']['airflow-latest'] }}" 21 | {% endif %} 22 | pullPolicy: IfNotPresent 23 | clusterConfig: 24 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 25 | vectorAggregatorConfigMapName: vector-aggregator-discovery 26 | {% endif %} 27 | credentialsSecret: test-airflow-credentials 28 | webservers: 29 | config: 30 | listenerClass: external-unstable 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | celeryExecutors: 37 | config: 38 | logging: 39 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 40 | roleGroups: 41 | default: 42 | replicas: 3 43 | schedulers: 44 | config: 45 | logging: 46 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 47 | roleGroups: 48 | default: 49 | replicas: 1 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | shmVolume: 18 | chmod: 19 | enabled: false 20 | 21 | auth: 22 | username: airflow 23 | password: airflow 24 | database: airflow 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | containerSecurityContext: 5 | runAsUser: auto 6 | 7 | master: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | replica: 18 | replicaCount: 1 19 | podSecurityContext: 20 | {% if test_scenario['values']['openshift'] == 'true' %} 21 | enabled: false 22 | {% else %} 23 | enabled: true 24 | {% endif %} 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | auth: 29 | password: redis 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/commons/health.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import logging 3 | import requests 4 | import sys 5 | import time 6 | 7 | if __name__ == "__main__": 8 | log_level = "DEBUG" 9 | logging.basicConfig( 10 | level=log_level, 11 | format="%(asctime)s %(levelname)s: %(message)s", 12 | stream=sys.stdout, 13 | ) 14 | 15 | try: 16 | role_group = sys.argv[1] 17 | except IndexError: 18 | role_group = "default" 19 | 20 | url = f"http://airflow-webserver-{role_group}:8080/api/v1/health" 21 | count = 0 22 | 23 | while True: 24 | try: 25 | count = count + 1 26 | res = requests.get(url, timeout=5) 27 | code = res.status_code 28 | if code == 200: 29 | break 30 | else: 31 | print( 32 | f"Got non 200 status code [{code}], retrying attempt no [{count}] ...." 33 | ) 34 | except requests.exceptions.Timeout: 35 | print(f"Connection timed out, retrying attempt no [{count}] ....") 36 | except requests.ConnectionError as e: 37 | print(f"Connection Error: {str(e)}") 38 | except requests.RequestException as e: 39 | print(f"General Error: {str(e)}") 40 | except Exception as e: 41 | print( 42 | f"General error occurred {str(e)}, retrying attempt no [{count}] ...." 43 | ) 44 | 45 | # Wait a little bit before retrying 46 | time.sleep(1) 47 | sys.exit(0) 48 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/10-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-redis-master 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: airflow-redis-replicas 28 | status: 29 | readyReplicas: 1 30 | replicas: 1 31 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/20-install-redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-redis 7 | --namespace $NAMESPACE 8 | --version 17.11.3 9 | -f helm-bitnami-redis-values.yaml 10 | --repo https://charts.bitnami.com/bitnami redis 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/30-listener-classes.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: | 6 | envsubst < listener-classes.yaml | kubectl apply -n $NAMESPACE -f - 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | commands: 6 | - script: > 7 | envsubst < install-airflow-cluster.yaml | 8 | kubectl apply -n $NAMESPACE -f - 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | resources: 17 | requests: 18 | memory: "128Mi" 19 | cpu: "100m" 20 | limits: 21 | memory: "128Mi" 22 | cpu: "400m" 23 | shmVolume: 24 | chmod: 25 | enabled: false 26 | 27 | auth: 28 | username: airflow 29 | password: airflow 30 | database: airflow 31 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | containerSecurityContext: 5 | runAsUser: auto 6 | 7 | master: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | resources: 17 | requests: 18 | memory: "128Mi" 19 | cpu: "200m" 20 | limits: 21 | memory: "128Mi" 22 | cpu: "800m" 23 | 24 | replica: 25 | replicaCount: 1 26 | podSecurityContext: 27 | {% if test_scenario['values']['openshift'] == 'true' %} 28 | enabled: false 29 | {% else %} 30 | enabled: true 31 | {% endif %} 32 | containerSecurityContext: 33 | enabled: false 34 | resources: 35 | requests: 36 | memory: "128Mi" 37 | cpu: "100m" 38 | limits: 39 | memory: "128Mi" 40 | cpu: "400m" 41 | 42 | auth: 43 | password: redis 44 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: test-airflow-credentials 6 | type: Opaque 7 | stringData: 8 | adminUser.username: airflow 9 | adminUser.firstname: Airflow 10 | adminUser.lastname: Admin 11 | adminUser.email: airflow@airflow.com 12 | adminUser.password: airflow 13 | connections.secretKey: thisISaSECRET_1234 14 | connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow 15 | connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow 16 | connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 17 | --- 18 | apiVersion: airflow.stackable.tech/v1alpha1 19 | kind: AirflowCluster 20 | metadata: 21 | name: airflow 22 | spec: 23 | image: 24 | {% if test_scenario['values']['airflow'].find(",") > 0 %} 25 | custom: "{{ test_scenario['values']['airflow'].split(',')[1] }}" 26 | productVersion: "{{ test_scenario['values']['airflow'].split(',')[0] }}" 27 | {% else %} 28 | productVersion: "{{ test_scenario['values']['airflow'] }}" 29 | {% endif %} 30 | pullPolicy: IfNotPresent 31 | clusterConfig: 32 | loadExamples: true 33 | credentialsSecret: test-airflow-credentials 34 | webservers: 35 | config: 36 | listenerClass: test-external-stable-$NAMESPACE 37 | roleGroups: 38 | default: 39 | replicas: 2 40 | external-unstable: 41 | replicas: 1 42 | config: 43 | listenerClass: test-external-unstable-$NAMESPACE 44 | cluster-internal: 45 | replicas: 1 46 | config: 47 | listenerClass: test-cluster-internal-$NAMESPACE 48 | celeryExecutors: 49 | roleGroups: 50 | default: 51 | replicas: 2 52 | schedulers: 53 | roleGroups: 54 | default: 55 | replicas: 1 56 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/listener-classes.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: listeners.stackable.tech/v1alpha1 3 | kind: ListenerClass 4 | metadata: 5 | name: test-cluster-internal-$NAMESPACE 6 | spec: 7 | serviceType: ClusterIP 8 | --- 9 | apiVersion: listeners.stackable.tech/v1alpha1 10 | kind: ListenerClass 11 | metadata: 12 | name: test-external-stable-$NAMESPACE 13 | spec: 14 | serviceType: NodePort 15 | --- 16 | apiVersion: listeners.stackable.tech/v1alpha1 17 | kind: ListenerClass 18 | metadata: 19 | name: test-external-unstable-$NAMESPACE 20 | spec: 21 | serviceType: NodePort 22 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/10-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | timeout: 600 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-openldap 6 | timeout: 300 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: openldap 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: create-ldap-user 6 | commands: 7 | - script: kubectl exec -n $NAMESPACE openldap-0 -- ldapsearch -H ldap://localhost:1389 -D cn=integrationtest,ou=users,dc=example,dc=org -w integrationtest -b ou=users,dc=example,dc=org > /dev/null 8 | - script: kubectl exec -n $NAMESPACE openldap-0 -- bash -c LDAPTLS_CACERT=/tls/ca.crt ldapsearch -Z -H ldaps://localhost:1636 -D cn=integrationtest,ou=users,dc=example,dc=org -w integrationtest -b ou=users,dc=example,dc=org > /dev/null 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/40-create-ldap-user.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: create-ldap-user 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ./create_ldap_user.sh openldap-0:/tmp 8 | - script: kubectl exec -n $NAMESPACE openldap-0 -- sh /tmp/create_ldap_user.sh 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/60-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 600 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | {% if test_scenario['values']['executor'] == 'celery' %} 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/70-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/70-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | stdin: true 22 | tty: true 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/80-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/80-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/90-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: metrics 6 | timeout: 480 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/90-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/metrics.py test-airflow-python-0:/tmp 8 | timeout: 480 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/create_ldap_user.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # To check the existing users 4 | # ldapsearch -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w admin -b ou=users,dc=example,dc=org 5 | 6 | # To check the new user 7 | # ldapsearch -H ldap://localhost:1389 -D cn=integrationtest,ou=users,dc=example,dc=org -w integrationtest -b ou=users,dc=example,dc=org 8 | 9 | cat << 'EOF' | ldapadd -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w admin 10 | dn: cn=integrationtest,ou=users,dc=example,dc=org 11 | objectClass: inetOrgPerson 12 | objectClass: posixAccount 13 | objectClass: shadowAccount 14 | cn: integrationtest 15 | uid: integrationtest 16 | givenName: Stackable 17 | sn: Integration-Test 18 | mail: integrationtest@stackable.de 19 | uidNumber: 16842 20 | gidNumber: 100 21 | homeDirectory: /home/integrationtest 22 | loginShell: /bin/bash 23 | userPassword: {crypt}x 24 | shadowLastChange: 0 25 | shadowMax: 0 26 | shadowWarning: 0 27 | EOF 28 | 29 | ldappasswd -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w admin -s integrationtest "cn=integrationtest,ou=users,dc=example,dc=org" 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | shmVolume: 18 | chmod: 19 | enabled: false 20 | 21 | auth: 22 | username: airflow 23 | password: airflow 24 | database: airflow 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | containerSecurityContext: 5 | runAsUser: auto 6 | 7 | master: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | replica: 18 | replicaCount: 1 19 | podSecurityContext: 20 | {% if test_scenario['values']['openshift'] == 'true' %} 21 | enabled: false 22 | {% else %} 23 | enabled: true 24 | {% endif %} 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | auth: 29 | password: redis 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-redis 6 | timeout: 360 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-redis-master 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-redis-replicas 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/10-install-redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-redis 7 | --namespace $NAMESPACE 8 | --version 17.11.3 9 | -f helm-bitnami-redis-values.yaml 10 | --repo https://charts.bitnami.com/bitnami redis 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: airflow-vector-aggregator 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/30-install-airflow-vector-aggregator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-vector-aggregator vector 7 | --namespace $NAMESPACE 8 | --version 0.42.1 9 | --repo https://helm.vector.dev 10 | --values airflow-vector-aggregator-values.yaml 11 | --- 12 | apiVersion: v1 13 | kind: ConfigMap 14 | metadata: 15 | name: airflow-vector-aggregator-discovery 16 | data: 17 | ADDRESS: airflow-vector-aggregator:6123 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/40-create-configmap-with-prepared-logs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: > 6 | kubectl create configmap prepared-logs 7 | --from-file=prepared-logs.py.json 8 | --namespace=$NAMESPACE 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/41-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 1200 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-automatic-log-config 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-webserver-custom-log-config 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | {% if test_scenario['values']['executor'] == 'celery' %} 24 | --- 25 | apiVersion: apps/v1 26 | kind: StatefulSet 27 | metadata: 28 | name: airflow-worker-automatic-log-config 29 | status: 30 | readyReplicas: 1 31 | replicas: 1 32 | --- 33 | apiVersion: apps/v1 34 | kind: StatefulSet 35 | metadata: 36 | name: airflow-worker-custom-log-config 37 | status: 38 | readyReplicas: 1 39 | replicas: 1 40 | {% endif %} 41 | --- 42 | apiVersion: apps/v1 43 | kind: StatefulSet 44 | metadata: 45 | name: airflow-scheduler-automatic-log-config 46 | status: 47 | readyReplicas: 1 48 | replicas: 1 49 | --- 50 | apiVersion: apps/v1 51 | kind: StatefulSet 52 | metadata: 53 | name: airflow-scheduler-custom-log-config 54 | status: 55 | readyReplicas: 1 56 | replicas: 1 57 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/50-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | stdin: true 22 | tty: true 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/51-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | - script: | 9 | kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py automatic-log-config 10 | kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py custom-log-config 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/51-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/52-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: metrics 6 | timeout: 600 7 | commands: 8 | - script: | 9 | kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py automatic-log-config 10 | kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py custom-log-config 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/52-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/metrics.py test-airflow-python-0:/tmp 8 | timeout: 240 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: >- 6 | kubectl exec --namespace=$NAMESPACE test-airflow-python-0 -- 7 | python /tmp/test_log_aggregation.py -n $NAMESPACE 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/60-test-log-aggregation.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: | 6 | kubectl cp ./test_log_aggregation.py $NAMESPACE/test-airflow-python-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | shmVolume: 18 | chmod: 19 | enabled: false 20 | 21 | auth: 22 | username: airflow 23 | password: airflow 24 | database: airflow 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | containerSecurityContext: 5 | runAsUser: auto 6 | 7 | master: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | replica: 18 | replicaCount: 1 19 | podSecurityContext: 20 | {% if test_scenario['values']['openshift'] == 'true' %} 21 | enabled: false 22 | {% else %} 23 | enabled: true 24 | {% endif %} 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | auth: 29 | password: redis 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/prepared-logs.py.json: -------------------------------------------------------------------------------- 1 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "DEBUG", "name": "TestLogger", "message": "Valid log event with log level DEBUG"} 2 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "INFO", "name": "TestLogger", "message": "Valid log event with log level INFO"} 3 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "WARNING", "name": "TestLogger", "message": "Valid log event with log level WARNING"} 4 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "ERROR", "name": "TestLogger", "message": "Valid log event with log level ERROR"} 5 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "CRITICAL", "name": "TestLogger", "message": "Valid log event with log level CRITICAL"} 6 | {"levelname": "INFO", "name": "TestLogger", "message": "Invalid log event without a timestamp"} 7 | {"asctime": "unparsable timestamp", "levelname": "INFO", "name": "TestLogger", "message": "Invalid log event with an unparsable timestamp"} 8 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "INFO", "message": "Invalid log event without a logger"} 9 | {"asctime": "2024-01-01 00:00:00,000", "name": "TestLogger", "message": "Invalid log event without a level"} 10 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "FATAL", "name": "TestLogger", "message": "Invalid log event with an unknown level"} 11 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "INFO", "name": "TestLogger"} 12 | "true" 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/test_log_aggregation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import requests 3 | 4 | 5 | def check_sent_events(): 6 | response = requests.post( 7 | "http://airflow-vector-aggregator:8686/graphql", 8 | json={ 9 | "query": """ 10 | { 11 | transforms(first:100) { 12 | nodes { 13 | componentId 14 | metrics { 15 | sentEventsTotal { 16 | sentEventsTotal 17 | } 18 | } 19 | } 20 | } 21 | } 22 | """ 23 | }, 24 | ) 25 | 26 | assert ( 27 | response.status_code == 200 28 | ), "Cannot access the API of the vector aggregator." 29 | 30 | result = response.json() 31 | 32 | transforms = result["data"]["transforms"]["nodes"] 33 | for transform in transforms: 34 | sentEvents = transform["metrics"]["sentEventsTotal"] 35 | componentId = transform["componentId"] 36 | 37 | if componentId == "filteredInvalidEvents": 38 | assert ( 39 | sentEvents is None or sentEvents["sentEventsTotal"] == 0 40 | ), "Invalid log events were sent." 41 | else: 42 | assert ( 43 | sentEvents is not None and sentEvents["sentEventsTotal"] > 0 44 | ), f'No events were sent in "{componentId}".' 45 | 46 | 47 | if __name__ == "__main__": 48 | check_sent_events() 49 | print("Test successful!") 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/10-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | timeout: 600 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/30-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 1200 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | {% if test_scenario['values']['executor'] == 'celery' %} 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | readyReplicas: 2 23 | replicas: 2 24 | {% endif %} 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/40-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | stdin: true 22 | tty: true 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/50-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: metrics 6 | timeout: 480 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/60-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/metrics.py test-airflow-python-0:/tmp 8 | timeout: 240 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | shmVolume: 18 | chmod: 19 | enabled: false 20 | 21 | auth: 22 | username: airflow 23 | password: airflow 24 | database: airflow 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | containerSecurityContext: 5 | runAsUser: auto 6 | 7 | master: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | replica: 18 | replicaCount: 1 19 | podSecurityContext: 20 | {% if test_scenario['values']['openshift'] == 'true' %} 21 | enabled: false 22 | {% else %} 23 | enabled: true 24 | {% endif %} 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | auth: 29 | password: redis 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/03-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/04-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/04-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | timeout: 600 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/05-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/05-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/10-clusterrole.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: airflow-spark-clusterrole 6 | rules: 7 | - apiGroups: 8 | - spark.stackable.tech 9 | resources: 10 | - sparkapplications 11 | verbs: 12 | - create 13 | - get 14 | - list 15 | --- 16 | apiVersion: rbac.authorization.k8s.io/v1 17 | kind: ClusterRoleBinding 18 | metadata: 19 | name: airflow-spark-clusterrole-binding 20 | roleRef: 21 | apiGroup: rbac.authorization.k8s.io 22 | kind: ClusterRole 23 | name: airflow-spark-clusterrole 24 | subjects: 25 | - apiGroup: rbac.authorization.k8s.io 26 | kind: Group 27 | name: system:serviceaccounts 28 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/30-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 1200 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | {% if test_scenario['values']['executor'] == 'celery' %} 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 30 5 | commands: 6 | 7 | {% if test_scenario['values']['executor'] == 'kubernetes' %} 8 | # check that the executor template configmap contains mounts and envs 9 | # will expect 4 (2 from from the volume declaration + mounts to two containers, base and gitsync) 10 | - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "test-cm-gitsync" | wc -l | grep 4 11 | # will expect 2 (two containers, base and gitsync) 12 | - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "AIRFLOW_TEST_VAR" | wc -l | grep 2 13 | # will expect 1 (one container, gitsync) 14 | - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "GITSYNC_USERNAME" | wc -l | grep 1 15 | - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "GITSYNC_PASSWORD" | wc -l | grep 1 16 | {% else %} 17 | # check that the statefulset contains mounts and envs 18 | # will expect 6 (2 from from the volume declaration + mounts to 3 containers, base and 2 gitsyncs, plus configmap restarter) 19 | - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "test-cm-gitsync" | wc -l | grep 6 20 | # will expect 3 (two containers, base and gitsync-1, and one initContainer gitsync-0) 21 | - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "AIRFLOW_TEST_VAR" | wc -l | grep 3 22 | # will expect 2 (one container, gitsync-1, and one initContainer gitsync-0) 23 | - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_USERNAME" | wc -l | grep 2 24 | - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_PASSWORD" | wc -l | grep 2 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/40-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | stdin: true 22 | tty: true 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/50-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: metrics 6 | timeout: 480 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/dag_metrics.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/60-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE dag_metrics.py test-airflow-python-0:/tmp 8 | timeout: 240 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/dag_metrics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import requests 4 | import time 5 | import sys 6 | import logging 7 | 8 | 9 | def assert_metric(role, metric): 10 | metric_response = requests.get( 11 | f"http://airflow-{role}-default-metrics:9102/metrics" 12 | ) 13 | assert metric_response.status_code == 200, ( 14 | f"Metrics could not be retrieved from the {role}." 15 | ) 16 | return metric in metric_response.text 17 | 18 | 19 | # Trigger a DAG run to create metrics 20 | dag_id = "sparkapp_dag" 21 | 22 | rest_url = "http://airflow-webserver-default:8080/api/v1" 23 | auth = ("airflow", "airflow") 24 | 25 | # allow a few moments for the DAGs to be registered to all roles 26 | time.sleep(10) 27 | 28 | response = requests.patch( 29 | f"{rest_url}/dags/{dag_id}", auth=auth, json={"is_paused": False} 30 | ) 31 | response = requests.post(f"{rest_url}/dags/{dag_id}/dagRuns", auth=auth, json={}) 32 | 33 | # Wait for the metrics to be consumed by the statsd-exporter 34 | time.sleep(5) 35 | # Test the DAG in a loop. Each time we call the script a new job will be started: we can avoid 36 | # or minimize this by looping over the check instead. 37 | iterations = 9 38 | loop = 0 39 | while True: 40 | try: 41 | logging.info(f"Response code: {response.status_code}") 42 | assert response.status_code == 200, "DAG run could not be triggered." 43 | # Worker is not deployed with the kubernetes executor so retrieve success metric from scheduler 44 | # (disable line-break flake checks) 45 | if (assert_metric("scheduler", "airflow_scheduler_heartbeat")) and ( 46 | assert_metric( 47 | "scheduler", "airflow_dagrun_duration_success_sparkapp_dag_count" 48 | ) 49 | ): # noqa: W503, W504 50 | break 51 | time.sleep(10) 52 | loop += 1 53 | if loop == iterations: 54 | logging.error("Still waiting for metrics. Exiting to start a new loop....") 55 | # force re-try of script 56 | sys.exit(1) 57 | except AssertionError as error: 58 | logging.warning(f"Encountered: {error}") 59 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | shmVolume: 18 | chmod: 19 | enabled: false 20 | 21 | auth: 22 | username: airflow 23 | password: airflow 24 | database: airflow 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | containerSecurityContext: 5 | runAsUser: auto 6 | 7 | master: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | replica: 18 | replicaCount: 1 19 | podSecurityContext: 20 | {% if test_scenario['values']['openshift'] == 'true' %} 21 | enabled: false 22 | {% else %} 23 | enabled: true 24 | {% endif %} 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | auth: 29 | password: redis 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/10-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | --values helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-keycloak 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: keycloak1 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: Deployment 18 | metadata: 19 | name: keycloak2 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/30-install-keycloak.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: | 6 | INSTANCE_NAME=keycloak1 \ 7 | REALM=test1 \ 8 | USERNAME=jane.doe \ 9 | FIRST_NAME=Jane \ 10 | LAST_NAME=Doe \ 11 | EMAIL=jane.doe@stackable.tech \ 12 | PASSWORD=T8mn72D9 \ 13 | CLIENT_ID=airflow1 \ 14 | CLIENT_SECRET=R1bxHUD569vHeQdw \ 15 | envsubst < install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - 16 | 17 | INSTANCE_NAME=keycloak2 \ 18 | REALM=test2 \ 19 | USERNAME=richard.roe \ 20 | FIRST_NAME=Richard \ 21 | LAST_NAME=Roe \ 22 | EMAIL=richard.roe@stackable.tech \ 23 | PASSWORD=NvfpU518 \ 24 | CLIENT_ID=airflow2 \ 25 | CLIENT_SECRET=scWzh0D4v0GN8NrN \ 26 | envsubst < install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - 27 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-airflow 6 | timeout: 1200 7 | commands: 8 | - script: > 9 | kubectl --namespace $NAMESPACE 10 | wait --for=condition=available=true 11 | airflowclusters.airflow.stackable.tech/airflow 12 | --timeout 301s 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: airflow-webserver-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: airflow-scheduler-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/40-install-airflow.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 300 5 | commands: 6 | - script: > 7 | envsubst '$NAMESPACE' < install-airflow.yaml | 8 | kubectl apply -n $NAMESPACE -f - 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-test-container 6 | timeout: 300 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/50-install-test-container.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: python 6 | --- 7 | kind: Role 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | metadata: 10 | name: python 11 | {% if test_scenario['values']['openshift'] == 'true' %} 12 | rules: 13 | - apiGroups: ["security.openshift.io"] 14 | resources: ["securitycontextconstraints"] 15 | resourceNames: ["privileged"] 16 | verbs: ["use"] 17 | {% endif %} 18 | --- 19 | kind: RoleBinding 20 | apiVersion: rbac.authorization.k8s.io/v1 21 | metadata: 22 | name: python 23 | subjects: 24 | - kind: ServiceAccount 25 | name: python 26 | roleRef: 27 | kind: Role 28 | name: python 29 | apiGroup: rbac.authorization.k8s.io 30 | --- 31 | apiVersion: kuttl.dev/v1beta1 32 | kind: TestStep 33 | metadata: 34 | name: install-test-container 35 | timeout: 300 36 | --- 37 | apiVersion: apps/v1 38 | kind: StatefulSet 39 | metadata: 40 | name: python 41 | labels: 42 | app: python 43 | spec: 44 | replicas: 1 45 | selector: 46 | matchLabels: 47 | app: python 48 | template: 49 | metadata: 50 | labels: 51 | app: python 52 | spec: 53 | serviceAccountName: python 54 | securityContext: 55 | fsGroup: 1000 56 | containers: 57 | - name: python 58 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 59 | stdin: true 60 | tty: true 61 | resources: 62 | requests: 63 | memory: "128Mi" 64 | cpu: "512m" 65 | limits: 66 | memory: "128Mi" 67 | cpu: "1" 68 | volumeMounts: 69 | - name: tls 70 | mountPath: /stackable/tls 71 | env: 72 | - name: REQUESTS_CA_BUNDLE 73 | value: /stackable/tls/ca.crt 74 | volumes: 75 | - name: tls 76 | csi: 77 | driver: secrets.stackable.tech 78 | volumeAttributes: 79 | secrets.stackable.tech/class: tls 80 | secrets.stackable.tech/scope: pod 81 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: login 6 | timeout: 300 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE python-0 -- python /stackable/login.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/60-login.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: login 6 | commands: 7 | - script: > 8 | envsubst '$NAMESPACE' < login.py | 9 | kubectl exec -n $NAMESPACE -i python-0 -- tee /stackable/login.py > /dev/null 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | resources: 17 | requests: 18 | memory: "128Mi" 19 | cpu: "512m" 20 | limits: 21 | memory: "128Mi" 22 | cpu: "1" 23 | 24 | shmVolume: 25 | chmod: 26 | enabled: false 27 | 28 | auth: 29 | username: airflow 30 | password: airflow 31 | database: airflow 32 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/10-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/11-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/11-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 12.5.6 9 | --values 11_helm-bitnami-postgresql-values.yaml 10 | --repo https://charts.bitnami.com/bitnami postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/11_helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | resources: 17 | requests: 18 | memory: "128Mi" 19 | cpu: "512m" 20 | limits: 21 | memory: "128Mi" 22 | cpu: "1" 23 | 24 | shmVolume: 25 | chmod: 26 | enabled: false 27 | 28 | auth: 29 | username: airflow 30 | password: airflow 31 | database: airflow 32 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/12-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/12-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl -n $NAMESPACE rollout status daemonset opa-server-default --timeout 300s 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/20-install-opa.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: install-opa 6 | --- 7 | apiVersion: opa.stackable.tech/v1alpha1 8 | kind: OpaCluster 9 | metadata: 10 | name: opa 11 | spec: 12 | image: 13 | {% if test_scenario['values']['opa-latest'].find(",") > 0 %} 14 | custom: "{{ test_scenario['values']['opa-latest'].split(',')[1] }}" 15 | productVersion: "{{ test_scenario['values']['opa-latest'].split(',')[0] }}" 16 | {% else %} 17 | productVersion: "{{ test_scenario['values']['opa-latest'] }}" 18 | {% endif %} 19 | pullPolicy: IfNotPresent 20 | clusterConfig: 21 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 22 | vectorAggregatorConfigMapName: vector-aggregator-discovery 23 | {% endif %} 24 | servers: 25 | config: 26 | logging: 27 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 28 | containers: 29 | opa: 30 | loggers: 31 | decision: 32 | level: INFO 33 | roleGroups: 34 | default: {} 35 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-airflow 6 | timeout: 1200 7 | commands: 8 | - script: > 9 | kubectl --namespace $NAMESPACE 10 | wait --for=condition=available=true 11 | airflowclusters.airflow.stackable.tech/airflow 12 | --timeout 301s 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: airflow-webserver-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: airflow-scheduler-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-test-container 6 | timeout: 300 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-runner 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/40-install-test-container.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: test-runner 6 | --- 7 | kind: Role 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | metadata: 10 | name: test-runner 11 | {% if test_scenario['values']['openshift'] == 'true' %} 12 | rules: 13 | - apiGroups: ["security.openshift.io"] 14 | resources: ["securitycontextconstraints"] 15 | resourceNames: ["privileged"] 16 | verbs: ["use"] 17 | {% endif %} 18 | --- 19 | kind: RoleBinding 20 | apiVersion: rbac.authorization.k8s.io/v1 21 | metadata: 22 | name: test-runner 23 | subjects: 24 | - kind: ServiceAccount 25 | name: test-runner 26 | roleRef: 27 | kind: Role 28 | name: test-runner 29 | apiGroup: rbac.authorization.k8s.io 30 | --- 31 | apiVersion: kuttl.dev/v1beta1 32 | kind: TestStep 33 | metadata: 34 | name: install-test-container 35 | timeout: 300 36 | --- 37 | apiVersion: apps/v1 38 | kind: StatefulSet 39 | metadata: 40 | name: test-runner 41 | labels: 42 | app: test-runner 43 | spec: 44 | replicas: 1 45 | selector: 46 | matchLabels: 47 | app: test-runner 48 | template: 49 | metadata: 50 | labels: 51 | app: test-runner 52 | spec: 53 | serviceAccountName: test-runner 54 | securityContext: 55 | fsGroup: 1000 56 | containers: 57 | - name: test-runner 58 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 59 | stdin: true 60 | tty: true 61 | resources: 62 | requests: 63 | memory: "128Mi" 64 | cpu: "512m" 65 | limits: 66 | memory: "128Mi" 67 | cpu: "1" 68 | volumeMounts: 69 | - name: tls 70 | mountPath: /stackable/tls 71 | env: 72 | - name: REQUESTS_CA_BUNDLE 73 | value: /stackable/tls/ca.crt 74 | volumes: 75 | - name: tls 76 | csi: 77 | driver: secrets.stackable.tech 78 | volumeAttributes: 79 | secrets.stackable.tech/class: tls 80 | secrets.stackable.tech/scope: pod 81 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/41-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: login 6 | timeout: 300 7 | commands: 8 | - script: > 9 | kubectl exec -n $NAMESPACE test-runner-0 -- 10 | python /stackable/check-authorization.py 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/41-check-authorization.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: login 6 | commands: 7 | - script: > 8 | kubectl cp 9 | 41_check-authorization.py 10 | $NAMESPACE/test-runner-0:/stackable/check-authorization.py 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-redis 6 | timeout: 360 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-redis-master 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-redis-replicas 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/10-install-redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-redis 7 | --namespace $NAMESPACE 8 | --version 17.11.3 9 | -f helm-bitnami-redis-values.yaml 10 | --repo https://charts.bitnami.com/bitnami redis 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 1200 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-worker-default 20 | status: 21 | readyReplicas: 2 22 | replicas: 2 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: airflow-scheduler-default 28 | status: 29 | readyReplicas: 1 30 | replicas: 1 31 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: remove-webserver 6 | timeout: 600 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/40-errors.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: airflow-webserver-default 6 | --- 7 | apiVersion: v1 8 | kind: Pod 9 | metadata: 10 | name: airflow-webserver-default-0 11 | --- 12 | apiVersion: v1 13 | kind: ConfigMap 14 | metadata: 15 | name: airflow-webserver-default 16 | --- 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: airflow-webserver-default 21 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/40-remove-webserver.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | webservers: null 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: change-worker-rolegroup 6 | timeout: 600 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-worker-newrolegroup 12 | --- 13 | apiVersion: v1 14 | kind: Pod 15 | metadata: 16 | name: airflow-worker-newrolegroup-0 17 | --- 18 | apiVersion: v1 19 | kind: ConfigMap 20 | metadata: 21 | name: airflow-worker-newrolegroup 22 | --- 23 | apiVersion: v1 24 | kind: Service 25 | metadata: 26 | name: airflow-worker-newrolegroup-metrics 27 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/50-change-worker-rolegroup.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | celeryExecutors: 8 | roleGroups: 9 | default: null 10 | newrolegroup: 11 | replicas: 1 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/50-errors.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: airflow-worker-default 6 | --- 7 | apiVersion: v1 8 | kind: Pod 9 | metadata: 10 | name: airflow-worker-default-0 11 | --- 12 | apiVersion: v1 13 | kind: Pod 14 | metadata: 15 | name: airflow-worker-default-1 16 | --- 17 | apiVersion: v1 18 | kind: ConfigMap 19 | metadata: 20 | name: airflow-worker-default 21 | --- 22 | apiVersion: v1 23 | kind: Service 24 | metadata: 25 | name: airflow-worker-default-metrics 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | shmVolume: 18 | chmod: 19 | enabled: false 20 | 21 | auth: 22 | username: airflow 23 | password: airflow 24 | database: airflow 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | containerSecurityContext: 5 | runAsUser: auto 6 | 7 | master: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | replica: 18 | replicaCount: 1 19 | podSecurityContext: 20 | {% if test_scenario['values']['openshift'] == 'true' %} 21 | enabled: false 22 | {% else %} 23 | enabled: true 24 | {% endif %} 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | auth: 29 | password: redis 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/07-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/07-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | --wait 13 | timeout: 600 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-available-condition 6 | timeout: 600 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-celery --timeout 301s 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/11-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | commands: 6 | # 7 | # Test envOverrides 8 | # 9 | - script: | 10 | kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' 11 | kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' 12 | kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' 13 | kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "credentialsSecret" and .value == "test-override")' 14 | 15 | 16 | - script: | 17 | kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' 18 | kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' 19 | kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' 20 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-available-condition 6 | timeout: 600 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-kubernetes --timeout 301s 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow-kubernetes 6 | spec: 7 | image: 8 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 9 | custom: "{{ test_scenario['values']['airflow-latest'].split(',')[1] }}" 10 | productVersion: "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 11 | {% else %} 12 | productVersion: "{{ test_scenario['values']['airflow-latest'] }}" 13 | {% endif %} 14 | clusterConfig: 15 | loadExamples: true 16 | exposeConfig: false 17 | credentialsSecret: airflow-credentials 18 | webservers: 19 | config: 20 | listenerClass: external-unstable 21 | roleGroups: 22 | default: 23 | replicas: 1 24 | config: 25 | resources: 26 | cpu: 27 | min: 410m 28 | podOverrides: 29 | spec: 30 | containers: 31 | - name: airflow 32 | resources: 33 | limits: 34 | cpu: 810m 35 | config: 36 | resources: 37 | cpu: 38 | min: 400m 39 | max: 800m 40 | memory: 41 | limit: 2Gi 42 | podOverrides: 43 | spec: 44 | containers: 45 | - name: airflow 46 | resources: 47 | limits: 48 | cpu: 750m 49 | requests: 50 | cpu: 350m 51 | kubernetesExecutors: 52 | config: 53 | resources: 54 | cpu: 55 | min: 400m 56 | max: 800m 57 | memory: 58 | limit: 2Gi 59 | podOverrides: 60 | spec: 61 | containers: 62 | - name: base 63 | resources: 64 | limits: 65 | cpu: 750m 66 | envOverrides: 67 | ROLE_VAR: role-value # there are no role groups for kubernetes executors 68 | AIRFLOW__METRICS__STATSD_ON: "False" # also set by the operator 69 | schedulers: 70 | roleGroups: 71 | default: 72 | replicas: 1 73 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/21-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 30 5 | commands: 6 | - script: | 7 | kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | yq -e '.spec.containers.[0].resources.limits | select (.cpu == "750m")' 8 | kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | yq -e '.spec.containers[] | select (.name == "base") | .env[] | select (.name == "AIRFLOW__METRICS__STATSD_ON" and .value == "False")' 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | resources: 17 | requests: 18 | memory: "128Mi" 19 | cpu: "100m" 20 | limits: 21 | memory: "128Mi" 22 | cpu: "400m" 23 | shmVolume: 24 | chmod: 25 | enabled: false 26 | 27 | auth: 28 | username: airflow 29 | password: airflow 30 | database: airflow 31 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | containerSecurityContext: 5 | runAsUser: auto 6 | 7 | master: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | resources: 17 | requests: 18 | memory: "128Mi" 19 | cpu: "200m" 20 | limits: 21 | memory: "128Mi" 22 | cpu: "800m" 23 | 24 | replica: 25 | replicaCount: 1 26 | podSecurityContext: 27 | {% if test_scenario['values']['openshift'] == 'true' %} 28 | enabled: false 29 | {% else %} 30 | enabled: true 31 | {% endif %} 32 | containerSecurityContext: 33 | enabled: false 34 | resources: 35 | requests: 36 | memory: "128Mi" 37 | cpu: "100m" 38 | limits: 39 | memory: "128Mi" 40 | cpu: "400m" 41 | 42 | auth: 43 | password: redis 44 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-redis 6 | timeout: 360 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-redis-master 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-redis-replicas 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/10-install-redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-redis 7 | --namespace $NAMESPACE 8 | --version 17.11.3 9 | -f helm-bitnami-redis-values.yaml 10 | --repo https://charts.bitnami.com/bitnami redis 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | shmVolume: 18 | chmod: 19 | enabled: false 20 | 21 | auth: 22 | username: airflow 23 | password: airflow 24 | database: airflow 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | containerSecurityContext: 5 | runAsUser: auto 6 | 7 | master: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | 17 | replica: 18 | replicaCount: 1 19 | podSecurityContext: 20 | {% if test_scenario['values']['openshift'] == 'true' %} 21 | enabled: false 22 | {% else %} 23 | enabled: true 24 | {% endif %} 25 | containerSecurityContext: 26 | enabled: false 27 | 28 | auth: 29 | password: redis 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/00-range-limit.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: LimitRange 4 | metadata: 5 | name: limit-request-ratio 6 | spec: 7 | limits: 8 | - type: "Container" 9 | maxLimitRequestRatio: 10 | cpu: 5 11 | memory: 1 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/10-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/20-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | --wait 13 | timeout: 600 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/30-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/30-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/40-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-available-condition 6 | timeout: 600 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow --timeout 301s 9 | --- 10 | apiVersion: kuttl.dev/v1beta1 11 | kind: TestAssert 12 | metadata: 13 | name: test-airflow-cluster 14 | timeout: 1200 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-webserver-default 20 | spec: 21 | template: 22 | spec: 23 | terminationGracePeriodSeconds: 120 24 | status: 25 | readyReplicas: 1 26 | replicas: 1 27 | {% if test_scenario['values']['executor'] == 'celery' %} 28 | --- 29 | apiVersion: apps/v1 30 | kind: StatefulSet 31 | metadata: 32 | name: airflow-worker-default 33 | spec: 34 | template: 35 | spec: 36 | terminationGracePeriodSeconds: 300 37 | status: 38 | readyReplicas: 2 39 | replicas: 2 40 | {% endif %} 41 | --- 42 | apiVersion: apps/v1 43 | kind: StatefulSet 44 | metadata: 45 | name: airflow-scheduler-default 46 | spec: 47 | template: 48 | spec: 49 | terminationGracePeriodSeconds: 120 50 | status: 51 | readyReplicas: 1 52 | replicas: 1 53 | --- 54 | apiVersion: policy/v1 55 | kind: PodDisruptionBudget 56 | metadata: 57 | name: airflow-webserver 58 | status: 59 | expectedPods: 1 60 | currentHealthy: 1 61 | disruptionsAllowed: 1 62 | {% if test_scenario['values']['executor'] == 'celery' %} 63 | --- 64 | apiVersion: policy/v1 65 | kind: PodDisruptionBudget 66 | metadata: 67 | name: airflow-worker 68 | status: 69 | expectedPods: 2 70 | currentHealthy: 2 71 | disruptionsAllowed: 1 72 | {% endif %} 73 | --- 74 | apiVersion: policy/v1 75 | kind: PodDisruptionBudget 76 | metadata: 77 | name: airflow-scheduler 78 | status: 79 | expectedPods: 1 80 | currentHealthy: 1 81 | disruptionsAllowed: 1 82 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/41-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | commands: 6 | # 7 | # Test envOverrides 8 | # 9 | - script: | 10 | set -eu 11 | 12 | # Config Test Data 13 | AIRFLOW_CONFIG=$( 14 | kubectl -n "$NAMESPACE" get cm airflow-webserver-default -o yaml \ 15 | | yq -e '.data["webserver_config.py"]' 16 | ) 17 | 18 | # Config Test Assertions 19 | echo "$AIRFLOW_CONFIG" | grep 'COMMON_HEADER_VAR = "group-value"' 20 | echo "$AIRFLOW_CONFIG" | grep 'ROLE_FOOTER_VAR = "role-value"' 21 | echo "$AIRFLOW_CONFIG" | grep -v 'ROLE_HEADER_VAR = "role-value"' 22 | echo "$AIRFLOW_CONFIG" | grep 'AUTH_ROLES_SYNC_AT_LOGIN = True' 23 | echo "$AIRFLOW_CONFIG" | grep 'AUTH_USER_REGISTRATION = False' 24 | echo "$AIRFLOW_CONFIG" | grep 'AUTH_USER_REGISTRATION_ROLE = "Rolegroup"' 25 | echo "$AIRFLOW_CONFIG" | grep 'OAUTH_PROVIDERS' 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/42-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # This test checks if the containerdebug-state.json file is present and valid 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | timeout: 600 6 | commands: 7 | - script: kubectl exec -n $NAMESPACE --container airflow airflow-scheduler-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' 8 | - script: kubectl exec -n $NAMESPACE --container airflow airflow-webserver-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/50-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | stdin: true 22 | tty: true 23 | resources: 24 | requests: 25 | memory: "128Mi" 26 | cpu: "100m" 27 | limits: 28 | memory: "128Mi" 29 | cpu: "400m" 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/60-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/70-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: metrics 6 | timeout: 480 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/70-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/metrics.py test-airflow-python-0:/tmp 8 | timeout: 240 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | securityContext: 5 | runAsUser: auto 6 | 7 | primary: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | resources: 17 | requests: 18 | memory: "128Mi" 19 | cpu: "100m" 20 | limits: 21 | memory: "128Mi" 22 | cpu: "400m" 23 | shmVolume: 24 | chmod: 25 | enabled: false 26 | 27 | auth: 28 | username: airflow 29 | password: airflow 30 | database: airflow 31 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | volumePermissions: 3 | enabled: false 4 | containerSecurityContext: 5 | runAsUser: auto 6 | 7 | master: 8 | podSecurityContext: 9 | {% if test_scenario['values']['openshift'] == 'true' %} 10 | enabled: false 11 | {% else %} 12 | enabled: true 13 | {% endif %} 14 | containerSecurityContext: 15 | enabled: false 16 | resources: 17 | requests: 18 | memory: "128Mi" 19 | cpu: "200m" 20 | limits: 21 | memory: "128Mi" 22 | cpu: "800m" 23 | 24 | replica: 25 | replicaCount: 1 26 | podSecurityContext: 27 | {% if test_scenario['values']['openshift'] == 'true' %} 28 | enabled: false 29 | {% else %} 30 | enabled: true 31 | {% endif %} 32 | containerSecurityContext: 33 | enabled: false 34 | resources: 35 | requests: 36 | memory: "128Mi" 37 | cpu: "100m" 38 | limits: 39 | memory: "128Mi" 40 | cpu: "400m" 41 | 42 | auth: 43 | password: redis 44 | --------------------------------------------------------------------------------