├── tests ├── templates │ ├── .gitkeep │ └── kuttl │ │ ├── orphaned-resources │ │ ├── 40-assert.yaml │ │ ├── 40-remove-webserver.yaml │ │ ├── 20-assert.yaml.j2 │ │ ├── 50-change-worker-rolegroup.yaml │ │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 05-assert.yaml │ │ ├── 10-install-redis.yaml │ │ ├── 05-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 40-errors.yaml │ │ ├── 10-assert.yaml │ │ ├── 50-errors.yaml │ │ ├── 50-assert.yaml │ │ ├── 30-assert.yaml │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ └── helm-bitnami-redis-values.yaml.j2 │ │ ├── external-access │ │ ├── 30-listener-classes.yaml │ │ ├── 40-install-airflow-cluster.yaml │ │ ├── 20-install-redis.yaml │ │ ├── 10-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── listener-classes.yaml │ │ ├── 20-assert.yaml │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ ├── 50-assert.yaml.j2 │ │ └── helm-bitnami-redis-values.yaml.j2 │ │ ├── logging │ │ ├── 60-test-log-aggregation.yaml │ │ ├── 60-assert.yaml │ │ ├── 51-health-check.yaml │ │ ├── 20-assert.yaml.j2 │ │ ├── 40-create-configmap-with-prepared-logs.yaml │ │ ├── 30-assert.yaml │ │ ├── 52-install-metrics-script.yaml │ │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 50-assert.yaml │ │ ├── 05-assert.yaml │ │ ├── 10-install-redis.yaml │ │ ├── 05-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 10-assert.yaml │ │ ├── 30-install-airflow-vector-aggregator.yaml │ │ ├── 50-install-airflow-python.yaml │ │ ├── 51-assert.yaml.j2 │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ ├── 52-assert.yaml.j2 │ │ ├── helm-bitnami-redis-values.yaml.j2 │ │ ├── 41-assert.yaml.j2 │ │ ├── prepared-logs.py.json │ │ ├── 70-assert.yaml.j2 │ │ └── test_log_aggregation.py │ │ ├── oidc │ │ ├── 60-login.yaml │ │ ├── 60-assert.yaml │ │ ├── 40-install-airflow.yaml │ │ ├── 20-assert.yaml.j2 │ │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 50-assert.yaml │ │ ├── 10-assert.yaml │ │ ├── 10-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 30-assert.yaml │ │ ├── 40-assert.yaml │ │ ├── 30-install-keycloak.yaml │ │ └── helm-bitnami-postgresql-values.yaml.j2 │ │ ├── opa │ │ ├── 20-assert.yaml │ │ ├── 12-assert.yaml.j2 │ │ ├── 41-assert.yaml │ │ ├── 12-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 40-assert.yaml │ │ ├── 11-assert.yaml │ │ ├── 10-patch-ns.yaml.j2 │ │ ├── 11-install-postgresql.yaml │ │ ├── 41-check-authorization.yaml.j2 │ │ ├── 30-assert.yaml │ │ ├── 32-create-users.yaml │ │ ├── 11_helm-bitnami-postgresql-values.yaml.j2 │ │ └── 20-install-opa.yaml.j2 │ │ ├── remote-logging │ │ ├── 03-assert.yaml │ │ ├── 60-health-check.yaml │ │ ├── 70-install-remote-logging-script.yaml │ │ ├── 10-assert.yaml │ │ ├── 50-assert.yaml │ │ ├── 00-range-limit.yaml.j2 │ │ ├── 03-setup-minio.yaml │ │ ├── 10-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 20-install-redis.yaml.j2 │ │ ├── 20-assert.yaml.j2 │ │ ├── 60-assert.yaml.j2 │ │ ├── 70-assert.yaml.j2 │ │ ├── 02-s3-secret.yaml │ │ ├── 50-install-airflow-python.yaml │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ └── helm-bitnami-redis-values.yaml.j2 │ │ ├── ldap │ │ ├── 80-health-check.yaml │ │ ├── 20-assert.yaml.j2 │ │ ├── 90-install-metrics-script.yaml │ │ ├── 30-assert.yaml │ │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 40-create-ldap-user.yaml │ │ ├── 05-assert.yaml │ │ ├── 70-assert.yaml │ │ ├── 05-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 10-install-redis.yaml.j2 │ │ ├── 10-assert.yaml.j2 │ │ ├── 40-assert.yaml │ │ ├── 90-assert.yaml.j2 │ │ ├── 80-assert.yaml.j2 │ │ ├── 70-install-airflow-python.yaml │ │ ├── 60-assert.yaml.j2 │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ ├── 95-assert.yaml.j2 │ │ ├── create_ldap_user.sh │ │ └── helm-bitnami-redis-values.yaml.j2 │ │ ├── mount-dags-gitsync │ │ ├── 60-assert.yaml │ │ ├── 60-install-metrics-script.yaml │ │ ├── 50-health-check.yaml │ │ ├── 05-assert.yaml.j2 │ │ ├── 05-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 03-assert.yaml │ │ ├── 40-assert.yaml │ │ ├── 03-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 04-install-redis.yaml.j2 │ │ ├── 04-assert.yaml.j2 │ │ ├── 50-assert.yaml.j2 │ │ ├── 40-install-airflow-python.yaml │ │ ├── 10-clusterrole.yaml │ │ ├── 30-assert.yaml.j2 │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ ├── 70-assert.yaml.j2 │ │ └── helm-bitnami-redis-values.yaml.j2 │ │ ├── smoke │ │ ├── 60-health-check.yaml │ │ ├── 30-assert.yaml.j2 │ │ ├── 70-install-metrics-script.yaml │ │ ├── 30-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 50-assert.yaml │ │ ├── 10-assert.yaml │ │ ├── 00-range-limit.yaml.j2 │ │ ├── 10-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 20-install-redis.yaml.j2 │ │ ├── 20-assert.yaml.j2 │ │ ├── 70-assert.yaml.j2 │ │ ├── 60-assert.yaml.j2 │ │ ├── 41-assert.yaml │ │ ├── 50-install-airflow-python.yaml │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ ├── 80-assert.yaml.j2 │ │ └── helm-bitnami-redis-values.yaml.j2 │ │ ├── triggerer │ │ ├── 50-health-check.yaml │ │ ├── 60-install-metrics-script.yaml │ │ ├── 05-assert.yaml │ │ ├── 40-assert.yaml │ │ ├── 05-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 10-install-redis.yaml.j2 │ │ ├── 10-assert.yaml.j2 │ │ ├── 50-assert.yaml.j2 │ │ ├── 60-assert.yaml.j2 │ │ ├── 40-install-airflow-python.yaml │ │ ├── 30-assert.yaml.j2 │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ ├── 70-assert.yaml.j2 │ │ └── helm-bitnami-redis-values.yaml.j2 │ │ ├── resources │ │ ├── 20-assert.yaml.j2 │ │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 05-assert.yaml │ │ ├── 10-install-redis.yaml │ │ ├── 05-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 10-assert.yaml │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ └── helm-bitnami-redis-values.yaml.j2 │ │ ├── mount-dags-configmap │ │ ├── 50-health-check.yaml │ │ ├── 20-assert.yaml.j2 │ │ ├── 60-install-metrics-script.yaml │ │ ├── 20-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 40-assert.yaml │ │ ├── 05-assert.yaml │ │ ├── 05-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 10-install-redis.yaml.j2 │ │ ├── 10-assert.yaml.j2 │ │ ├── 60-assert.yaml.j2 │ │ ├── 50-assert.yaml.j2 │ │ ├── 40-install-airflow-python.yaml │ │ ├── 30-assert.yaml.j2 │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ ├── 70-assert.yaml.j2 │ │ └── helm-bitnami-redis-values.yaml.j2 │ │ ├── cluster-operation │ │ ├── 06-assert.yaml.j2 │ │ ├── 09-assert.yaml │ │ ├── 06-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 02-assert.yaml │ │ ├── 31-assert.yaml │ │ ├── 04-install-redis.yaml │ │ ├── 02-install-postgresql.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 04-assert.yaml │ │ ├── 20-assert.yaml │ │ ├── 30-assert.yaml │ │ ├── 08-assert.yaml │ │ ├── 10-assert.yaml │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ ├── helm-bitnami-redis-values.yaml.j2 │ │ ├── 20-stop-airflow.yaml.j2 │ │ └── 10-pause-airflow.yaml.j2 │ │ └── overrides │ │ ├── 10-assert.yaml │ │ ├── 20-assert.yaml │ │ ├── 05-assert.yaml │ │ ├── 05-install-postgresql.yaml │ │ ├── 07-install-redis.yaml.j2 │ │ ├── 07-assert.yaml.j2 │ │ ├── 21-assert.yaml │ │ ├── helm-bitnami-postgresql-values.yaml.j2 │ │ └── helm-bitnami-redis-values.yaml.j2 ├── interu.yaml ├── release.yaml └── kuttl-test.yaml.jinja2 ├── docs ├── antora.yml ├── modules │ └── airflow │ │ ├── partials │ │ ├── hardware-requirements.adoc │ │ ├── supported-versions.adoc │ │ └── nav.adoc │ │ ├── images │ │ ├── airflow_dags.png │ │ ├── airflow_login.png │ │ ├── airflow_dag_log.png │ │ ├── airflow_running.png │ │ ├── airflow_security.png │ │ ├── airflow_dag_graph.png │ │ ├── airflow_dag_s3_logs.png │ │ ├── airflow_connection_ui.png │ │ ├── airflow_security_ldap.png │ │ ├── airflow_dag_log_opensearch.png │ │ ├── airflow_edit_s3_connection.png │ │ └── getting_started │ │ │ ├── airflow_dags.png │ │ │ ├── airflow_login.png │ │ │ ├── airflow_pods.png │ │ │ └── airflow_running.png │ │ ├── examples │ │ ├── getting_started │ │ │ └── code │ │ │ │ ├── test_getting_started_helm.sh │ │ │ │ ├── test_getting_started_stackablectl.sh │ │ │ │ ├── install_output.txt │ │ │ │ ├── install_output.txt.j2 │ │ │ │ ├── airflow-credentials.yaml │ │ │ │ └── airflow.yaml │ │ ├── example-airflow-spark-clusterrole.yaml │ │ ├── example-pyspark-pi.yaml │ │ ├── example-airflow-spark-clusterrolebinding.yaml │ │ ├── example-airflow-secret.yaml │ │ ├── example-airflow-kubernetes-executor-s3-logging.yaml │ │ ├── example-airflow-gitsync.yaml │ │ ├── example-airflow-dags-configmap.yaml │ │ └── example-airflow-kubernetes-executor-s3-xcom.yaml │ │ └── pages │ │ ├── reference │ │ ├── crds.adoc │ │ ├── index.adoc │ │ └── commandline-parameters.adoc │ │ ├── usage-guide │ │ ├── index.adoc │ │ ├── monitoring.adoc │ │ ├── operations │ │ │ ├── cluster-operations.adoc │ │ │ ├── index.adoc │ │ │ ├── pod-placement.adoc │ │ │ └── pod-disruptions.adoc │ │ ├── listenerclass.adoc │ │ ├── db-init.adoc │ │ └── logging.adoc │ │ ├── required-external-components.adoc │ │ └── getting_started │ │ └── index.adoc └── templating_vars.yaml ├── scripts ├── run_tests.sh ├── render_readme.sh ├── generate-manifests.sh └── docs_templating.sh ├── rust └── operator-binary │ ├── src │ ├── operations │ │ ├── mod.rs │ │ └── graceful_shutdown.rs │ └── util.rs │ ├── build.rs │ └── Cargo.toml ├── .gitattributes ├── deploy ├── stackable-operators-ns.yaml ├── helm │ ├── ct.yaml │ └── airflow-operator │ │ ├── templates │ │ ├── configmap.yaml │ │ ├── service.yaml │ │ ├── _maintenance.tpl │ │ └── serviceaccount.yaml │ │ ├── Chart.yaml │ │ ├── configs │ │ └── properties.yaml │ │ ├── .helmignore │ │ └── README.md ├── config-spec │ └── properties.yaml └── DO_NOT_EDIT.md ├── .actionlint.yaml ├── .github ├── actionlint.yaml ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── normal-issue.md │ ├── 01-normal-issue.md │ └── new_version.md ├── workflows │ ├── general_daily_security.yml │ └── pr_pre-commit.yaml └── PULL_REQUEST_TEMPLATE │ ├── pre-release-getting-started-script.md │ └── pre-release-rust-deps.md ├── rust-toolchain.toml ├── .readme ├── static │ └── borrowed │ │ └── stackable_overview.png ├── partials │ ├── borrowed │ │ ├── header.md.j2 │ │ ├── related_reading.md.j2 │ │ ├── overview_blurb.md.j2 │ │ ├── documentation.md.j2 │ │ └── links.md.j2 │ └── main.md.j2 └── README.md.j2 ├── .envrc.sample ├── nix ├── meta.json └── README.md ├── .vscode ├── settings.json └── launch.json ├── .pylintrc ├── .gitignore ├── renovate.json ├── .dockerignore ├── .hadolint.yaml ├── rustfmt.toml ├── .yamllint.yaml ├── .markdownlint.yaml ├── examples └── simple-airflow-cluster.yaml ├── Cargo.toml └── shell.nix /tests/templates/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/antora.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: home 3 | version: "nightly" 4 | -------------------------------------------------------------------------------- /scripts/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ./scripts/run-tests "$@" 4 | -------------------------------------------------------------------------------- /rust/operator-binary/src/operations/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod graceful_shutdown; 2 | pub mod pdb; 3 | -------------------------------------------------------------------------------- /rust/operator-binary/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | built::write_built_file().unwrap(); 3 | } 4 | -------------------------------------------------------------------------------- /docs/modules/airflow/partials/hardware-requirements.adoc: -------------------------------------------------------------------------------- 1 | * 0.2 cores (e.g. i5 or similar) 2 | * 256MB RAM 3 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | nix/** linguist-generated 2 | Cargo.nix linguist-generated 3 | crate-hashes.json linguist-generated 4 | -------------------------------------------------------------------------------- /deploy/stackable-operators-ns.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: stackable-operators 6 | -------------------------------------------------------------------------------- /.actionlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | self-hosted-runner: 3 | # Ubicloud machines we are using 4 | labels: 5 | - ubicloud-standard-8-arm 6 | -------------------------------------------------------------------------------- /.github/actionlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | self-hosted-runner: 3 | # Ubicloud machines we are using 4 | labels: 5 | - ubicloud-standard-8-arm 6 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | # DO NOT EDIT, this file is generated by operator-templating 2 | [toolchain] 3 | channel = "1.89.0" 4 | profile = "default" 5 | -------------------------------------------------------------------------------- /.readme/static/borrowed/stackable_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/.readme/static/borrowed/stackable_overview.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_dags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_dags.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_login.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_dag_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_dag_log.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_running.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_running.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_security.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_security.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_dag_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_dag_graph.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_dag_s3_logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_dag_s3_logs.png -------------------------------------------------------------------------------- /.envrc.sample: -------------------------------------------------------------------------------- 1 | # vim: syntax=conf 2 | # 3 | # If you use direnv, you can autoload the nix shell: 4 | # You will need to allow the directory the first time. 5 | use nix 6 | -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_connection_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_connection_ui.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_security_ldap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_security_ldap.png -------------------------------------------------------------------------------- /nix/meta.json: -------------------------------------------------------------------------------- 1 | {"operator": {"name": "airflow-operator", "pretty_string": "Apache Airflow", "product_string": "airflow", "url": "stackabletech/airflow-operator.git"}} 2 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: remove-webserver 6 | timeout: 600 7 | -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_dag_log_opensearch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_dag_log_opensearch.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/airflow_edit_s3_connection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/airflow_edit_s3_connection.png -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/test_getting_started_helm.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "$0")" 5 | ./getting_started.sh helm 6 | -------------------------------------------------------------------------------- /docs/modules/airflow/images/getting_started/airflow_dags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/getting_started/airflow_dags.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/getting_started/airflow_login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/getting_started/airflow_login.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/getting_started/airflow_pods.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/getting_started/airflow_pods.png -------------------------------------------------------------------------------- /docs/modules/airflow/images/getting_started/airflow_running.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/airflow-operator/HEAD/docs/modules/airflow/images/getting_started/airflow_running.png -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/test_getting_started_stackablectl.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "$0")" 5 | ./getting_started.sh stackablectl 6 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/reference/crds.adoc: -------------------------------------------------------------------------------- 1 | = CRD Reference 2 | 3 | Find all CRD reference for the Stackable Operator for Apache Airfow at: {crd-docs-base-url}/airflow-operator/{crd-docs-version}. 4 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/40-remove-webserver.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | webservers: null 8 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rust-analyzer.rustfmt.overrideCommand": [ 3 | "rustfmt", 4 | "+nightly-2025-10-23", 5 | "--edition", 6 | "2024", 7 | "--" 8 | ], 9 | } 10 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/install_output.txt: -------------------------------------------------------------------------------- 1 | Installed commons=0.0.0-dev operator 2 | Installed secret=0.0.0-dev operator 3 | Installed listener=0.0.0-dev operator 4 | Installed airflow=0.0.0-dev operator 5 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/30-listener-classes.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: | 6 | envsubst < listener-classes.yaml | kubectl apply -n $NAMESPACE -f - 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/60-test-log-aggregation.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: | 6 | kubectl cp ./test_log_aggregation.py $NAMESPACE/test-airflow-python-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/60-login.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: login 6 | commands: 7 | - script: | 8 | kubectl cp -n $NAMESPACE login.py python-0:/stackable/login.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl -n $NAMESPACE wait --for=condition=available --timeout=10m opacluster/test-opa 7 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | 3 | # These rules are for missing docstrings which doesn't matter much for most of our simple scripts 4 | disable=C0114,C0115,C0116 5 | 6 | [FORMAT] 7 | 8 | max-line-length=999 9 | indent-string=' ' 10 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/header.md.j2: -------------------------------------------------------------------------------- 1 | 2 |

3 | Stackable Logo 4 |

5 | 6 |

{{title}}

7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: login 6 | timeout: 300 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE python-0 -- python /stackable/login.py 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tests/_work/ 2 | debug/ 3 | target/ 4 | **/*.rs.bk 5 | 6 | .idea/ 7 | *.iws 8 | *.iml 9 | 10 | *.tgz 11 | 12 | result 13 | image.tar 14 | 15 | tilt_options.json 16 | 17 | .direnv/ 18 | .direnvrc 19 | .envrc 20 | 21 | .DS_Store 22 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/40-install-airflow.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 300 5 | commands: 6 | - script: > 7 | envsubst '$NAMESPACE' < install-airflow.yaml | 8 | kubectl apply -n $NAMESPACE -f - 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 900 5 | --- 6 | apiVersion: apps/v1 7 | kind: Deployment 8 | metadata: 9 | name: test-minio 10 | status: 11 | readyReplicas: 1 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: >- 6 | kubectl exec --namespace=$NAMESPACE test-airflow-python-0 -- 7 | python /tmp/test_log_aggregation.py -n $NAMESPACE 8 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/related_reading.md.j2: -------------------------------------------------------------------------------- 1 | 2 | {%- if related_reading_links -%} 3 | ## Related Reading 4 | {% for (text, link) in related_reading_links %} 5 | * [{{text}}]({{link}}) 6 | {%- endfor %} 7 | {%- endif -%} 8 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/index.adoc: -------------------------------------------------------------------------------- 1 | = Usage guide 2 | :description: Practical instructions to make the most out of the Stackable operator for Apache Airflow. 3 | 4 | Practical instructions to make the most out of the Stackable operator for Apache Airflow. 5 | -------------------------------------------------------------------------------- /docs/templating_vars.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | helm: 3 | repo_name: sdp-charts 4 | repo_url: oci.stackable.tech 5 | versions: 6 | commons: 0.0.0-dev 7 | secret: 0.0.0-dev 8 | listener: 0.0.0-dev 9 | airflow: 0.0.0-dev 10 | postgresql: 16.5.0 11 | redis: 20.11.3 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 600 5 | commands: 6 | - script: > 7 | envsubst < install-airflow-cluster.yaml | 8 | kubectl apply -n $NAMESPACE -f - 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/80-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/51-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/60-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: metrics 6 | timeout: 480 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/dag_metrics.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/60-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/12-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/50-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/60-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE triggerer_metrics.py test-airflow-python-0:/tmp 8 | timeout: 240 9 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/install_output.txt.j2: -------------------------------------------------------------------------------- 1 | Installed commons={{ versions.commons }} operator 2 | Installed secret={{ versions.secret }} operator 3 | Installed listener={{ versions.listener }} operator 4 | Installed airflow={{ versions.airflow }} operator 5 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/60-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE dag_metrics.py test-airflow-python-0:/tmp 8 | timeout: 240 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/60-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/30-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/40-create-configmap-with-prepared-logs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: > 6 | kubectl create configmap prepared-logs 7 | --from-file=prepared-logs.py.json 8 | --namespace=$NAMESPACE 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/50-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/50-health-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 480 5 | commands: 6 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp 7 | timeout: 240 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/06-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: airflow-vector-aggregator 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/05-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/90-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/metrics.py test-airflow-python-0:/tmp 8 | timeout: 480 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/41-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: login 6 | timeout: 300 7 | commands: 8 | - script: > 9 | kubectl exec -n $NAMESPACE test-runner-0 -- 10 | python /stackable/check-authorization.py 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/70-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/metrics.py test-airflow-python-0:/tmp 8 | timeout: 240 9 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/monitoring.adoc: -------------------------------------------------------------------------------- 1 | = Monitoring 2 | :description: Airflow instances export Prometheus metrics for monitoring. 3 | 4 | The managed Airflow instances are automatically configured to export Prometheus metrics. 5 | See xref:operators:monitoring.adoc[] for more details. 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/52-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/metrics.py test-airflow-python-0:/tmp 8 | timeout: 240 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/70-install-remote-logging-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: remote-logging 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE remote-logging-metrics.py test-airflow-python-0:/tmp 8 | timeout: 240 9 | -------------------------------------------------------------------------------- /deploy/helm/ct.yaml: -------------------------------------------------------------------------------- 1 | # This file is used for chart-testing (https://github.com/helm/chart-testing) 2 | # The name "ct.yaml" is not very self-descriptive but it is the default that chart-testing is looking for 3 | --- 4 | remote: origin 5 | target-branch: main 6 | chart-dirs: 7 | - deploy/helm 8 | all: true 9 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | data: 4 | {{ (.Files.Glob "configs/*").AsConfig | indent 2 }} 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ include "operator.fullname" . }}-configmap 8 | labels: 9 | {{- include "operator.labels" . | nindent 4 }} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/60-install-metrics-script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: metrics 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/metrics.py test-airflow-python-0:/tmp 8 | timeout: 240 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-openldap 6 | timeout: 300 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: openldap 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/12-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/operations/cluster-operations.adoc: -------------------------------------------------------------------------------- 1 | = Cluster Operation 2 | 3 | Airflow installations can be configured with different cluster operations like pausing reconciliation or stopping the cluster. See xref:concepts:operations/cluster_operations.adoc[cluster operations] for more details. 4 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/09-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # For this assert we expect the database operation to be logged 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | timeout: 30 6 | commands: 7 | - script: | 8 | kubectl -n $NAMESPACE logs airflow-scheduler-default-0 | grep "Database migrating done!" 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-test-container 6 | timeout: 300 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/50-change-worker-rolegroup.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | celeryExecutors: 8 | roleGroups: 9 | default: null 10 | newrolegroup: 11 | replicas: 1 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/30-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/40-create-ldap-user.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: create-ldap-user 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ./create_ldap_user.sh openldap-0:/tmp 8 | - script: kubectl exec -n $NAMESPACE openldap-0 -- sh /tmp/create_ldap_user.sh 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-test-container 6 | timeout: 300 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-runner 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-available-condition 6 | timeout: 600 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-celery --timeout 301s 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-available-condition 6 | timeout: 600 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-kubernetes --timeout 301s 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/06-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/70-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/05-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/11-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/20-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-spark-clusterrole.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: airflow-spark-clusterrole 6 | rules: 7 | - apiGroups: 8 | - spark.stackable.tech 9 | resources: 10 | - sparkapplications 11 | verbs: 12 | - create 13 | - get 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/31-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # For this step we expect the database operation to NOT be logged 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | timeout: 30 6 | commands: 7 | - script: | 8 | kubectl -n $NAMESPACE logs airflow-scheduler-default-0 | grep -q "Database migrating done!" && exit 1 || exit 0 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-python 6 | timeout: 240 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-airflow-python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/00-range-limit.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: v1 4 | kind: LimitRange 5 | metadata: 6 | name: limit-request-ratio 7 | spec: 8 | limits: 9 | - type: "Container" 10 | maxLimitRequestRatio: 11 | cpu: 5 12 | memory: 1 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v2 3 | name: airflow-operator 4 | version: "0.0.0-dev" 5 | appVersion: "0.0.0-dev" 6 | description: The Stackable Operator for Apache Airflow 7 | home: https://github.com/stackabletech/airflow-operator 8 | maintainers: 9 | - name: Stackable 10 | url: https://www.stackable.tech 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "local>stackabletech/.github:renovate-config" 5 | ], 6 | "ignorePaths": [".github/workflows/build.yaml", ".github/workflows/general_daily_security.yml", ".github/workflows/integration-test.yml", ".github/workflows/pr_pre-commit.yaml"] 7 | } 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/00-range-limit.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: v1 4 | kind: LimitRange 5 | metadata: 6 | name: limit-request-ratio 7 | spec: 8 | limits: 9 | - type: "Container" 10 | maxLimitRequestRatio: 11 | cpu: 5 12 | memory: 1 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /docs/modules/airflow/partials/supported-versions.adoc: -------------------------------------------------------------------------------- 1 | // The version ranges supported by Airflow-Operator 2 | // This is a separate file, since it is used by both the direct Airflow-Operator documentation, and the overarching 3 | // Stackable Platform documentation. 4 | 5 | - 3.0.6 (LTS) 6 | - 3.0.1 (experimental) 7 | - 2.10.5 (deprecated) 8 | - 2.9.3 (deprecated) 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/10-install-redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-redis 7 | --namespace $NAMESPACE 8 | --version 17.11.3 9 | -f helm-bitnami-redis-values.yaml 10 | --repo https://charts.bitnami.com/bitnami redis 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-pyspark-pi.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: spark.stackable.tech/v1alpha1 3 | kind: SparkApplication 4 | metadata: 5 | name: pyspark-pi 6 | spec: 7 | sparkImage: 8 | productVersion: 3.5.7 9 | mode: cluster 10 | mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py 11 | executor: 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/operations/index.adoc: -------------------------------------------------------------------------------- 1 | = Operations 2 | 3 | This section of the documentation is intended for the operations teams that maintain a Stackable Data Platform installation. 4 | 5 | Read the xref:concepts:operations/index.adoc[concepts page on operations] that contains the necessary details to operate the platform in a production environment. 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/03-setup-minio.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install test-minio 7 | --namespace $NAMESPACE 8 | --version 17.0.19 9 | -f helm-bitnami-minio-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/minio 11 | timeout: 240 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/10-install-redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-redis 7 | --namespace $NAMESPACE 8 | --version 17.11.3 9 | -f helm-bitnami-redis-values.yaml 10 | --repo https://charts.bitnami.com/bitnami redis 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/04-install-redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-redis 7 | --namespace $NAMESPACE 8 | --version 17.11.3 9 | -f helm-bitnami-redis-values.yaml 10 | --repo https://charts.bitnami.com/bitnami redis 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/overview_blurb.md.j2: -------------------------------------------------------------------------------- 1 | 2 | It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka, Apache Druid, Trino or Apache Spark, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms). 3 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/10-install-redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-redis 7 | --namespace $NAMESPACE 8 | --version 17.11.3 9 | -f helm-bitnami-redis-values.yaml 10 | --repo https://charts.bitnami.com/bitnami redis 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/02-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/20-install-redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-redis 7 | --namespace $NAMESPACE 8 | --version 17.11.3 9 | -f helm-bitnami-redis-values.yaml 10 | --repo https://charts.bitnami.com/bitnami redis 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/03-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | debug/ 2 | target/ 3 | **/*.rs.bk 4 | 5 | .idea/ 6 | *.iws 7 | 8 | Cargo.nix 9 | crate-hashes.json 10 | result 11 | image.tar 12 | 13 | # We do NOT want to ignore .git because we use the `built` crate to gather the current git commit hash at built time 14 | # This means we need the .git directory in our Docker image, it will be thrown away and won't be included in the final image 15 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/reference/index.adoc: -------------------------------------------------------------------------------- 1 | = Reference 2 | 3 | Consult the reference documentation section to find exhaustive information on: 4 | 5 | * Descriptions and default values of all properties in the CRDs used by this operator in the xref:reference/crds.adoc[]. 6 | * The xref:reference/commandline-parameters.adoc[] and xref:reference/environment-variables.adoc[] accepted by the operator. 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | timeout: 600 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/10-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/10-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | --values helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/05-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/10-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/10-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/11-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 12.5.6 9 | --values 11_helm-bitnami-postgresql-values.yaml 10 | --repo https://charts.bitnami.com/bitnami postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/10-install-postgresql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-postgresql 7 | --namespace $NAMESPACE 8 | --version 16.4.2 9 | -f helm-bitnami-postgresql-values.yaml 10 | oci://registry-1.docker.io/bitnamicharts/postgresql 11 | --wait 12 | timeout: 600 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-spark-clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRoleBinding 4 | metadata: 5 | name: airflow-spark-clusterrole-binding 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: ClusterRole 9 | name: airflow-spark-clusterrole 10 | subjects: 11 | - apiGroup: rbac.authorization.k8s.io 12 | kind: Group 13 | name: system:serviceaccounts 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/10-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | timeout: 600 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/41-check-authorization.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: login 6 | commands: 7 | - script: > 8 | kubectl cp 9 | {% if test_scenario['values']['airflow'].startswith("2") %} 10 | 41_check-authorization_2.py 11 | {% else %} 12 | 41_check-authorization_3.py 13 | {% endif %} 14 | $NAMESPACE/test-runner-0:/stackable/check-authorization.py 15 | -------------------------------------------------------------------------------- /scripts/render_readme.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Check if jinja2 is there 5 | if ! command -v jinja2 &> /dev/null 6 | then 7 | echo "jinja2 could not be found. Use 'pip install jinja2-cli' to install it." 8 | exit 1 9 | fi 10 | 11 | SCRIPT_DIR=$(dirname "$0") 12 | cd "$SCRIPT_DIR/../.readme" 13 | jinja2 README.md.j2 -o ../README.md 14 | cd .. 15 | 16 | python3 scripts/ensure_one_trailing_newline.py README.md 17 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/10-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | timeout: 600 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-keycloak 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: keycloak1 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: Deployment 18 | metadata: 19 | name: keycloak2 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/10-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | timeout: 600 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/04-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | timeout: 600 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/40-errors.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: airflow-webserver-default 6 | --- 7 | apiVersion: v1 8 | kind: Pod 9 | metadata: 10 | name: airflow-webserver-default-0 11 | --- 12 | apiVersion: v1 13 | kind: ConfigMap 14 | metadata: 15 | name: airflow-webserver-default 16 | --- 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: airflow-webserver-default 21 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/20-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | --wait 13 | timeout: 600 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/07-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | --wait 13 | timeout: 600 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/20-install-redis.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | - script: >- 7 | helm install airflow-redis 8 | --namespace $NAMESPACE 9 | --version 17.11.3 10 | -f helm-bitnami-redis-values.yaml 11 | --repo https://charts.bitnami.com/bitnami redis 12 | --wait 13 | timeout: 600 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | blank_issues_enabled: true 3 | contact_links: 4 | - name: 🙋🏾 Question 5 | about: Use this to ask a question about this project 6 | url: https://github.com/orgs/stackabletech/discussions/new?category=q-a 7 | - name: 🚀 Feature Requests and other things 8 | about: Open an issue with your feature request or any other issue not covered elsewhere 9 | url: https://github.com/stackabletech/airflow-operator/issues/new 10 | -------------------------------------------------------------------------------- /deploy/config-spec/properties.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | spec: 3 | units: [] 4 | properties: 5 | - property: &credentialsSecret 6 | propertyNames: 7 | - name: "credentialsSecret" 8 | kind: 9 | type: "env" 10 | datatype: 11 | type: "string" 12 | roles: 13 | - name: "node" 14 | required: true 15 | asOfVersion: "0.0.0" 16 | description: "The secret where the Airflow credentials are stored." 17 | -------------------------------------------------------------------------------- /.hadolint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ignored: 3 | # Warning: Use the -y switch to avoid manual input dnf install -y 4 | # https://github.com/hadolint/hadolint/wiki/DL3038 5 | # Reason: We set `assumeyes=True` in dnf.conf in our base image 6 | - DL3038 7 | 8 | # Warning: Specify version with dnf install -y - 9 | # https://github.com/hadolint/hadolint/wiki/DL3041 10 | # Reason: It's good advice, but we're not set up to pin versions just yet 11 | - DL3041 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-redis 6 | timeout: 360 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-redis-master 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-redis-replicas 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-redis 6 | timeout: 360 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-redis-master 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-redis-replicas 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /deploy/DO_NOT_EDIT.md: -------------------------------------------------------------------------------- 1 | # DO NOT EDIT 2 | 3 | These Helm charts and manifests are automatically generated. 4 | Please do not edit anything except for files explicitly mentioned below in this 5 | directory manually. 6 | 7 | The following files are ok to edit: 8 | 9 | - helm/airflow-operator/templates/roles.yaml 10 | - helm/airflow-operator/values.yaml 11 | 12 | The details are in-motion but check this repository for a few details: 13 | 14 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/configs/properties.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | spec: 3 | units: [] 4 | properties: 5 | - property: &credentialsSecret 6 | propertyNames: 7 | - name: "credentialsSecret" 8 | kind: 9 | type: "env" 10 | datatype: 11 | type: "string" 12 | roles: 13 | - name: "node" 14 | required: true 15 | asOfVersion: "0.0.0" 16 | description: "The secret where the Airflow credentials are stored." 17 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # This file includes unstable features, so you need to run "cargo +nightly fmt" to format your code. 2 | # It's also ok to use the stable toolchain by simple running "cargo fmt", but using the nigthly formatter is prefered. 3 | 4 | # https://doc.rust-lang.org/nightly/edition-guide/rust-2024/rustfmt-style-edition.html 5 | style_edition = "2024" 6 | imports_granularity = "Crate" 7 | group_imports = "StdExternalCrate" 8 | reorder_impl_items = true 9 | use_field_init_shorthand = true 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/04-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-redis 6 | timeout: 360 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-redis-master 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-redis-replicas 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-redis 6 | timeout: 360 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-redis-master 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-redis-replicas 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /.yamllint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | ignore: | 5 | deploy/helm/**/templates 6 | 7 | rules: 8 | line-length: disable 9 | truthy: 10 | check-keys: false 11 | comments: 12 | min-spaces-from-content: 1 # Needed due to https://github.com/adrienverge/yamllint/issues/443 13 | indentation: 14 | indent-sequences: consistent 15 | comments-indentation: disable # This is generally useless and interferes with commented example values 16 | braces: 17 | max-spaces-inside: 1 18 | max-spaces-inside-empty: 0 19 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/30-install-airflow-vector-aggregator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install airflow-vector-aggregator vector 7 | --namespace $NAMESPACE 8 | --version 0.45.0 9 | --repo https://helm.vector.dev 10 | --values airflow-vector-aggregator-values.yaml 11 | --- 12 | apiVersion: v1 13 | kind: ConfigMap 14 | metadata: 15 | name: airflow-vector-aggregator-discovery 16 | data: 17 | ADDRESS: airflow-vector-aggregator:6123 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/50-errors.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: airflow-worker-default 6 | --- 7 | apiVersion: v1 8 | kind: Pod 9 | metadata: 10 | name: airflow-worker-default-0 11 | --- 12 | apiVersion: v1 13 | kind: Pod 14 | metadata: 15 | name: airflow-worker-default-1 16 | --- 17 | apiVersion: v1 18 | kind: ConfigMap 19 | metadata: 20 | name: airflow-worker-default 21 | --- 22 | apiVersion: v1 23 | kind: Service 24 | metadata: 25 | name: airflow-worker-default-metrics 26 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "lldb", 6 | "request": "launch", 7 | "name": "Debug operator binary", 8 | "cargo": { 9 | "args": ["build"], 10 | "filter": { 11 | "name": "stackable-{[ operator.name }]", 12 | "kind": "bin" 13 | } 14 | }, 15 | "args": ["run"], 16 | "cwd": "${workspaceFolder}" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/07-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/04-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['executor'] == 'celery' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | metadata: 6 | name: test-airflow-redis 7 | timeout: 360 8 | --- 9 | apiVersion: apps/v1 10 | kind: StatefulSet 11 | metadata: 12 | name: airflow-redis-master 13 | status: 14 | readyReplicas: 1 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-redis-replicas 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/70-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: metrics 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --airflow-version "{{ test_scenario['values']['airflow'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --airflow-version "{{ test_scenario['values']['airflow'] }}" 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /tests/interu.yaml: -------------------------------------------------------------------------------- 1 | runners: 2 | amd64: 3 | platform: aks-1.32 4 | ttl: 6h 5 | node-groups: 6 | - name: default 7 | arch: amd64 8 | size: medium 9 | disk-gb: 100 10 | nodes: 3 11 | 12 | profiles: 13 | # TODO (@Techassi): This will be enabled later 14 | # schedule: 15 | # strategy: use-runner 16 | # runner: amd64 17 | # options: 18 | # beku-parallelism: 2 19 | smoke-latest: 20 | strategy: use-runner 21 | runner: amd64 22 | options: 23 | beku-parallelism: 2 24 | beku-test-suite: smoke-latest 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: create-ldap-user 6 | commands: 7 | - script: kubectl exec -n $NAMESPACE openldap-0 -- ldapsearch -H ldap://localhost:1389 -D cn=integrationtest,ou=users,dc=example,dc=org -w integrationtest -b ou=users,dc=example,dc=org > /dev/null 8 | - script: kubectl exec -n $NAMESPACE openldap-0 -- bash -c LDAPTLS_CACERT=/tls/ca.crt ldapsearch -Z -H ldaps://localhost:1636 -D cn=integrationtest,ou=users,dc=example,dc=org -w integrationtest -b ou=users,dc=example,dc=org > /dev/null 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/90-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: metrics 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --airflow-version "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --airflow-version "{{ test_scenario['values']['airflow-latest'] }}" 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/operations/pod-placement.adoc: -------------------------------------------------------------------------------- 1 | = Pod placement 2 | 3 | You can configure the Pod placement of the Airflow pods as described in xref:concepts:operations/pod_placement.adoc[]. 4 | 5 | The default affinities created by the operator are: 6 | 7 | 1. Co-locate all the Airflow Pods (weight 20) 8 | 2. Distribute all Pods within the same role (worker, webserver, scheduler) (weight 70) 9 | 10 | == Kubernetes executors 11 | 12 | The Kubernetes executors are no different, they try to co-locate with other Airflow Pods and (with a higher priority) distribute all executors. 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/listener-classes.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: listeners.stackable.tech/v1alpha1 3 | kind: ListenerClass 4 | metadata: 5 | name: test-cluster-internal-$NAMESPACE 6 | spec: 7 | serviceType: ClusterIP 8 | --- 9 | apiVersion: listeners.stackable.tech/v1alpha1 10 | kind: ListenerClass 11 | metadata: 12 | name: test-external-stable-$NAMESPACE 13 | spec: 14 | serviceType: NodePort 15 | --- 16 | apiVersion: listeners.stackable.tech/v1alpha1 17 | kind: ListenerClass 18 | metadata: 19 | name: test-external-unstable-$NAMESPACE 20 | spec: 21 | serviceType: NodePort 22 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/60-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow'] }}" 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: change-worker-rolegroup 6 | timeout: 600 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-worker-newrolegroup 12 | --- 13 | apiVersion: v1 14 | kind: Pod 15 | metadata: 16 | name: airflow-worker-newrolegroup-0 17 | --- 18 | apiVersion: v1 19 | kind: ConfigMap 20 | metadata: 21 | name: airflow-worker-newrolegroup 22 | --- 23 | apiVersion: v1 24 | kind: Service 25 | metadata: 26 | name: airflow-worker-newrolegroup-metrics 27 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/60-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: metrics 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --airflow-version "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --airflow-version "{{ test_scenario['values']['airflow-latest'] }}" 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/80-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'] }}" 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/50-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'] }}" 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /tests/release.yaml: -------------------------------------------------------------------------------- 1 | # Contains all operators required to run the test suite. 2 | --- 3 | releases: 4 | # Do not change the name of the release as it's referenced from run_tests.sh 5 | tests: 6 | releaseDate: 1970-01-01 7 | description: Integration test 8 | products: 9 | commons: 10 | operatorVersion: 0.0.0-dev 11 | secret: 12 | operatorVersion: 0.0.0-dev 13 | listener: 14 | operatorVersion: 0.0.0-dev 15 | airflow: 16 | operatorVersion: 0.0.0-dev 17 | opa: 18 | operatorVersion: 0.0.0-dev 19 | spark-k8s: 20 | operatorVersion: 0.0.0-dev 21 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/60-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'] }}" 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/60-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: triggerer_metrics 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/triggerer_metrics.py --airflow-version "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/triggerer_metrics.py --airflow-version "{{ test_scenario['values']['airflow-latest'] }}" 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/50-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'] }}" 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/50-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'] }}" 12 | {% endif %} 13 | 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/70-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: remote-logging 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 9 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/remote-logging-metrics.py --airflow-version "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 10 | {% else %} 11 | - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/remote-logging-metrics.py --airflow-version "{{ test_scenario['values']['airflow-latest'] }}" 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/70-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | imagePullPolicy: IfNotPresent 22 | stdin: true 23 | tty: true 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/50-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | imagePullPolicy: IfNotPresent 22 | stdin: true 23 | tty: true 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-postgresql 6 | timeout: 480 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-postgresql 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-redis-master 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: airflow-redis-replicas 28 | status: 29 | readyReplicas: 1 30 | replicas: 1 31 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/21-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 30 5 | commands: 6 | - script: | 7 | set -eu 8 | 9 | kubectl -n $NAMESPACE get cm airflow-kubernetes-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | yq -e '.spec.containers.[0].resources.limits | select (.cpu == "750m")' 10 | kubectl -n $NAMESPACE get cm airflow-kubernetes-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | yq -e '.spec.containers[] | select (.name == "base") | .env[] | select (.name == "AIRFLOW__METRICS__STATSD_ON" and .value == "False")' 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/40-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | imagePullPolicy: IfNotPresent 22 | stdin: true 23 | tty: true 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/40-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | imagePullPolicy: IfNotPresent 22 | stdin: true 23 | tty: true 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/40-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | imagePullPolicy: IfNotPresent 22 | stdin: true 23 | tty: true 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 1200 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-worker-default 20 | status: 21 | readyReplicas: 2 22 | replicas: 2 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: airflow-scheduler-default 28 | status: 29 | readyReplicas: 1 30 | replicas: 1 31 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # ============= 2 | # This file is automatically generated from the templates in stackabletech/operator-templating 3 | # DON'T MANUALLY EDIT THIS FILE 4 | # ============= 5 | 6 | # Patterns to ignore when building packages. 7 | # This supports shell glob matching, relative path matching, and 8 | # negation (prefixed with !). Only one pattern per line. 9 | .DS_Store 10 | # Common VCS dirs 11 | .git/ 12 | .gitignore 13 | .bzr/ 14 | .bzrignore 15 | .hg/ 16 | .hgignore 17 | .svn/ 18 | # Common backup files 19 | *.swp 20 | *.bak 21 | *.tmp 22 | *.orig 23 | *~ 24 | # Various IDEs 25 | .project 26 | .idea/ 27 | *.tmproj 28 | .vscode/ 29 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/templates/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | # Note(@sbernauer): We could also call the Service something like 7 | # "product-operator-conversion-webhook". However, in the future we will have more webhooks, and 8 | # it seems like an overkill to have a dedicated Service per webhook. 9 | name: {{ include "operator.fullname" . }} 10 | labels: 11 | {{- include "operator.labels" . | nindent 4 }} 12 | spec: 13 | selector: 14 | {{- include "operator.selectorLabels" . | nindent 6 }} 15 | ports: 16 | - name: conversion-webhook 17 | protocol: TCP 18 | port: 8443 19 | targetPort: 8443 20 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-airflow 6 | timeout: 1200 7 | commands: 8 | - script: > 9 | kubectl --namespace $NAMESPACE 10 | wait --for=condition=available=true 11 | airflowclusters.airflow.stackable.tech/airflow 12 | --timeout 301s 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: airflow-webserver-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: airflow-scheduler-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-airflow 6 | timeout: 1200 7 | commands: 8 | - script: > 9 | kubectl --namespace $NAMESPACE 10 | wait --for=condition=available=true 11 | airflowclusters.airflow.stackable.tech/airflow 12 | --timeout 301s 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: airflow-webserver-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: airflow-scheduler-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/41-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | commands: 6 | # 7 | # Test envOverrides 8 | # 9 | - script: | 10 | set -eu 11 | 12 | # Config Test Data 13 | AIRFLOW_CONFIG=$( 14 | kubectl -n "$NAMESPACE" get cm airflow-webserver-default -o yaml \ 15 | | yq -e '.data["webserver_config.py"]' 16 | ) 17 | 18 | # Config Test Assertions 19 | echo "$AIRFLOW_CONFIG" | grep 'COMMON_HEADER_VAR = "group-value"' 20 | echo "$AIRFLOW_CONFIG" | grep 'ROLE_FOOTER_VAR = "role-value"' 21 | echo "$AIRFLOW_CONFIG" | grep -v 'ROLE_HEADER_VAR = "role-value"' 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/normal-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Normal issue 3 | about: This is just a normal empty issue with a simple checklist 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Issue checklist 11 | 12 | This is a simple checklist of things to bear in mind when creating a new issue. 13 | 14 | - [ ] Describe the use-case, as far is possible. For instance, using the pattern "As a XXXX, I would like XXXX to be able to do XXXX" helps to identify the feature as well as the problem it is intended to address. 15 | - [ ] Indicate an approximate level of importance and urgency. 16 | - [ ] Indicate if there is a known work-around until such time as the issue has been implemented. 17 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/51-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-webserver-health-check 6 | timeout: 480 7 | commands: 8 | {% if test_scenario['values']['airflow'].find(",") > 0 %} 9 | {% set airflow_version = test_scenario['values']['airflow'].split(',')[0] %} 10 | {% else %} 11 | {% set airflow_version = test_scenario['values']['airflow'] %} 12 | {% endif %} 13 | - script: | 14 | kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ airflow_version }}" 15 | kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ airflow_version }}" 16 | -------------------------------------------------------------------------------- /.readme/README.md.j2: -------------------------------------------------------------------------------- 1 | {%- set title="Stackable Operator for Apache Airflow" -%} 2 | {%- set operator_name="airflow" -%} 3 | {%- set operator_docs_slug="airflow" -%} 4 | {%- set related_reading_links=[] -%} 5 | 6 | {% filter trim %} 7 | {%- include "partials/borrowed/header.md.j2" -%} 8 | {% endfilter %} 9 | 10 | {% filter trim %} 11 | {%- include "partials/borrowed/links.md.j2" -%} 12 | {% endfilter %} 13 | 14 | {% filter trim %} 15 | {%- include "partials/main.md.j2" -%} 16 | {% endfilter %} 17 | 18 | {% filter trim %} 19 | {%- include "partials/borrowed/footer.md.j2" -%} 20 | {% endfilter %} 21 | 22 | {% filter trim %} 23 | {%- include "partials/borrowed/related_reading.md.j2" -%} 24 | {% endfilter %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-stop-airflow 6 | timeout: 180 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=stopped airflowclusters.airflow.stackable.tech/airflow --timeout 301s 9 | --- 10 | apiVersion: apps/v1 11 | kind: StatefulSet 12 | metadata: 13 | name: airflow-webserver-default 14 | status: 15 | replicas: 0 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | replicas: 0 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: airflow-scheduler-default 28 | status: 29 | replicas: 0 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/60-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 600 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | {% if test_scenario['values']['executor'] == 'celery' %} 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-restart-airflow 6 | timeout: 600 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow --timeout 301s 9 | --- 10 | apiVersion: apps/v1 11 | kind: StatefulSet 12 | metadata: 13 | name: airflow-webserver-default 14 | status: 15 | replicas: 1 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | replicas: 3 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: airflow-scheduler-default 28 | status: 29 | replicas: 1 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/30-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 1200 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | {% if test_scenario['values']['executor'] == 'celery' %} 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /.github/workflows/general_daily_security.yml: -------------------------------------------------------------------------------- 1 | # ============= 2 | # This file is automatically generated from the templates in stackabletech/operator-templating 3 | # DON'T MANUALLY EDIT THIS FILE 4 | # ============= 5 | --- 6 | name: Daily Security Audit 7 | 8 | on: 9 | schedule: 10 | - cron: '15 4 * * *' 11 | workflow_dispatch: 12 | 13 | permissions: {} 14 | 15 | jobs: 16 | audit: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 20 | with: 21 | persist-credentials: false 22 | - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 23 | with: 24 | token: ${{ secrets.GITHUB_TOKEN }} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/10-clusterrole.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: airflow-spark-clusterrole 6 | rules: 7 | - apiGroups: 8 | - spark.stackable.tech 9 | resources: 10 | - sparkapplications 11 | verbs: 12 | - create 13 | - get 14 | - list 15 | --- 16 | apiVersion: rbac.authorization.k8s.io/v1 17 | kind: ClusterRoleBinding 18 | metadata: 19 | name: airflow-spark-clusterrole-binding 20 | roleRef: 21 | apiGroup: rbac.authorization.k8s.io 22 | kind: ClusterRole 23 | name: airflow-spark-clusterrole 24 | subjects: 25 | - apiGroup: rbac.authorization.k8s.io 26 | kind: Group 27 | name: system:serviceaccounts 28 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/30-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 1200 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | {% if test_scenario['values']['executor'] == 'celery' %} 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | readyReplicas: 2 23 | replicas: 2 24 | {% endif %} 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/30-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 1200 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-default 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | {% if test_scenario['values']['executor'] == 'celery' %} 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: airflow-worker-default 21 | status: 22 | readyReplicas: 1 23 | replicas: 1 24 | {% endif %} 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /rust/operator-binary/src/util.rs: -------------------------------------------------------------------------------- 1 | use stackable_operator::k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector}; 2 | 3 | pub fn env_var_from_secret(var_name: &str, secret: &str, secret_key: &str) -> EnvVar { 4 | EnvVar { 5 | name: String::from(var_name), 6 | value_from: Some(EnvVarSource { 7 | secret_key_ref: Some(SecretKeySelector { 8 | name: String::from(secret), 9 | key: String::from(secret_key), 10 | ..Default::default() 11 | }), 12 | ..Default::default() 13 | }), 14 | ..Default::default() 15 | } 16 | } 17 | 18 | pub fn role_service_name(name: &str, role: &str) -> String { 19 | format!("{name}-{role}") 20 | } 21 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/templates/_maintenance.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Create a list of maintenance related env vars. 3 | */}} 4 | {{- define "maintenance.envVars" -}} 5 | {{- with .Values.maintenance }} 6 | {{- if not .endOfSupportCheck.enabled }} 7 | - name: EOS_DISABLED 8 | value: "true" 9 | {{- end }} 10 | {{- if and .endOfSupportCheck.enabled .endOfSupportCheck.mode }} 11 | - name: EOS_CHECK_MODE 12 | value: {{ .endOfSupportCheck.mode }} 13 | {{ end }} 14 | {{- if and .endOfSupportCheck.enabled .endOfSupportCheck.interval }} 15 | - name: EOS_INTERVAL 16 | value: {{ .endOfSupportCheck.interval }} 17 | {{ end }} 18 | {{- if not .customResourceDefinitions.maintain }} 19 | - name: DISABLE_CRD_MAINTENANCE 20 | value: "true" 21 | {{- end }} 22 | {{- end }} 23 | {{- end }} 24 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-secret.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: simple-airflow-credentials 6 | type: Opaque 7 | stringData: 8 | adminUser.username: airflow 9 | adminUser.firstname: Airflow 10 | adminUser.lastname: Admin 11 | adminUser.email: airflow@airflow.com 12 | adminUser.password: airflow 13 | connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 14 | # Only needed when using celery workers (instead of Kubernetes executors) 15 | connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 16 | connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 17 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/reference/commandline-parameters.adoc: -------------------------------------------------------------------------------- 1 | = Command Line Parameters 2 | 3 | This operator accepts the following command line parameters: 4 | 5 | == product-config 6 | 7 | *Default value*: `/etc/stackable/airflow-operator/config-spec/properties.yaml` 8 | 9 | *Required*: false 10 | 11 | *Multiple values:* false 12 | 13 | [source] 14 | ---- 15 | stackable-airflow-operator run --product-config /foo/bar/properties.yaml 16 | ---- 17 | 18 | == watch-namespace 19 | 20 | *Default value*: All namespaces 21 | 22 | *Required*: false 23 | 24 | *Multiple values:* false 25 | 26 | The operator **only** watches for resources in the provided namespace `test`: 27 | 28 | [source] 29 | ---- 30 | stackable-airflow-operator run --watch-namespace test 31 | ---- 32 | -------------------------------------------------------------------------------- /nix/README.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | # Updating nix dependencies 7 | 8 | ## Run the following for an operator 9 | 10 | > [!NOTE] 11 | > We track the `master` branch of crate2nix as that is relatively up to date, but the releases are infrequent. 12 | 13 | ```shell 14 | niv update crate2nix 15 | niv update nixpkgs 16 | niv update beku.py -b X.Y.Z # Using the release tag 17 | ``` 18 | 19 | ### Test 20 | 21 | - Run make `regenerate-nix` to ensure crate2nix works 22 | - Run a smoke test to ensure beku.py works. 23 | - Run `make run-dev` to ensure nixpkgs are fine. 24 | 25 | ## Update operator-templating 26 | 27 | Do the same as above, but from `template/` 28 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/02-s3-secret.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: minio-credentials 6 | labels: 7 | secrets.stackable.tech/class: spark-pi-private-s3-credentials-class 8 | timeout: 240 9 | stringData: 10 | accessKey: minioAccessKey 11 | secretKey: minioSecretKey 12 | # The following two entries are used by the Bitnami chart for MinIO to 13 | # set up credentials for accessing buckets managed by the MinIO tenant. 14 | root-user: minioAccessKey 15 | root-password: minioSecretKey 16 | --- 17 | apiVersion: secrets.stackable.tech/v1alpha1 18 | kind: SecretClass 19 | metadata: 20 | name: spark-pi-private-s3-credentials-class 21 | spec: 22 | backend: 23 | k8sSearch: 24 | searchNamespace: 25 | pod: {} 26 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/documentation.md.j2: -------------------------------------------------------------------------------- 1 | 2 | ## Documentation 3 | 4 | The stable documentation for this operator can be found in our [Stackable Data Platform documentation](https://docs.stackable.tech/home/stable/{{operator_docs_slug}}). 5 | If you are interested in the most recent state of this repository, check out the [nightly docs](https://docs.stackable.tech/home/nightly/{{operator_docs_slug}}) instead. 6 | 7 | The documentation for all Stackable products can be found at [docs.stackable.tech](https://docs.stackable.tech). 8 | 9 | If you have a question about the Stackable Data Platform, contact us via our [homepage](https://stackable.tech/) or ask a public question in our [Discussions forum](https://github.com/orgs/stackabletech/discussions). 10 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: simple-airflow-credentials 6 | type: Opaque 7 | stringData: 8 | adminUser.username: airflow 9 | adminUser.firstname: Airflow 10 | adminUser.lastname: Admin 11 | adminUser.email: airflow@airflow.com 12 | adminUser.password: airflow 13 | connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 14 | # Only needed when using celery workers (instead of Kubernetes executors) 15 | connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 16 | connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 17 | -------------------------------------------------------------------------------- /.readme/partials/main.md.j2: -------------------------------------------------------------------------------- 1 | This is a Kubernetes operator to manage [Apache Airflow](https://airflow.apache.org/) ensembles. 2 | 3 | {% filter trim %} 4 | {%- include "partials/borrowed/overview_blurb.md.j2" -%} 5 | {% endfilter %} 6 | 7 | ## Installation 8 | 9 | You can install the operator using [stackablectl or helm](https://docs.stackable.tech/home/stable/{{operator_name}}/getting_started/installation). 10 | 11 | Read on to get started with it, or see it in action in one of our [demos](https://stackable.tech/en/demos/). 12 | 13 | ## Getting Started 14 | 15 | You can follow this [tutorial](https://docs.stackable.tech/home/stable/{{operator_name}}/getting_started/first_steps). 16 | 17 | {% filter trim %} 18 | {%- include "partials/borrowed/documentation.md.j2" -%} 19 | {% endfilter %} 20 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/32-create-users.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: create-users 6 | timeout: 300 7 | commands: 8 | - script: | 9 | kubectl exec -n $NAMESPACE airflow-webserver-default-0 -- airflow users create \ 10 | --username "jane.doe" \ 11 | --firstname "Jane" \ 12 | --lastname "Doe" \ 13 | --email "jane.doe@stackable.tech" \ 14 | --password "T8mn72D9" \ 15 | --role "User" 16 | 17 | kubectl exec -n $NAMESPACE airflow-webserver-default-0 -- airflow users create \ 18 | --username "richard.roe" \ 19 | --firstname "Richard" \ 20 | --lastname "Roe" \ 21 | --email "richard.roe@stackable.tech" \ 22 | --password "NvfpU518" \ 23 | --role "User" 24 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/required-external-components.adoc: -------------------------------------------------------------------------------- 1 | = Required external components 2 | :description: Airflow requires PostgreSQL for database support, and Redis for Celery executors. 3 | :airflow-prerequisites: https://airflow.apache.org/docs/apache-airflow/stable/installation/prerequisites.html 4 | 5 | Airflow requires an SQL database to operate. 6 | The {airflow-prerequisites}[Airflow documentation] specifies: 7 | 8 | Fully supported for production usage: 9 | 10 | * PostgreSQL: 12, 13, 14, 15, 16 11 | 12 | NOTE: The SDP Airflow images do not bundle the MySQL provider for Airflow. 13 | If you need MySQL or MariaDB as an Airflow back-end, you will need to create a custom image to include this provider. 14 | 15 | The Celery exectutor also requires: 16 | 17 | * Redis (any stable version) 18 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/links.md.j2: -------------------------------------------------------------------------------- 1 | 2 | [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/stackabletech/{{operator_name}}-operator/graphs/commit-activity) 3 | [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-green.svg)](https://docs.stackable.tech/home/stable/contributor/index.html) 4 | [![License OSL3.0](https://img.shields.io/badge/license-OSL3.0-green)](./LICENSE) 5 | 6 | [Documentation](https://docs.stackable.tech/home/stable/{{operator_docs_slug}}) {% if quickstart_link %}| [Quickstart]({{quickstart_link}}) {% endif %}| [Stackable Data Platform](https://stackable.tech/) | [Platform Docs](https://docs.stackable.tech/) | [Discussions](https://github.com/orgs/stackabletech/discussions) | [Discord](https://discord.gg/7kZ3BNnCAF) 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/08-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-install-airflow 6 | timeout: 1200 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow --timeout 301s 9 | --- 10 | apiVersion: apps/v1 11 | kind: StatefulSet 12 | metadata: 13 | name: airflow-webserver-default 14 | status: 15 | readyReplicas: 1 16 | replicas: 1 17 | --- 18 | apiVersion: apps/v1 19 | kind: StatefulSet 20 | metadata: 21 | name: airflow-worker-default 22 | status: 23 | readyReplicas: 2 24 | replicas: 2 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-pause-airflow 6 | timeout: 180 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=reconciliationPaused airflowclusters.airflow.stackable.tech/airflow --timeout 301s 9 | --- 10 | apiVersion: apps/v1 11 | kind: StatefulSet 12 | metadata: 13 | name: airflow-webserver-default 14 | status: 15 | readyReplicas: 1 16 | replicas: 1 17 | --- 18 | apiVersion: apps/v1 19 | kind: StatefulSet 20 | metadata: 21 | name: airflow-worker-default 22 | status: 23 | readyReplicas: 2 24 | replicas: 2 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: airflow-scheduler-default 30 | status: 31 | readyReplicas: 1 32 | replicas: 1 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | 30 | shmVolume: 31 | chmod: 32 | enabled: false 33 | 34 | auth: 35 | username: airflow 36 | password: airflow 37 | database: airflow 38 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | 30 | shmVolume: 31 | chmod: 32 | enabled: false 33 | 34 | auth: 35 | username: airflow 36 | password: airflow 37 | database: airflow 38 | -------------------------------------------------------------------------------- /scripts/generate-manifests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This script reads a Helm chart from deploy/helm/airflow-operator and 3 | # generates manifest files into deploy/manifestss 4 | set -e 5 | 6 | tmp=$(mktemp -d ./manifests-XXXXX) 7 | 8 | helm template --output-dir "$tmp" \ 9 | --include-crds \ 10 | --name-template airflow-operator \ 11 | deploy/helm/airflow-operator 12 | 13 | for file in "$tmp"/airflow-operator/*/*; do 14 | yq eval -i 'del(.. | select(has("app.kubernetes.io/managed-by")) | ."app.kubernetes.io/managed-by")' /dev/stdin < "$file" 15 | yq eval -i 'del(.. | select(has("helm.sh/chart")) | ."helm.sh/chart")' /dev/stdin < "$file" 16 | sed -i '/# Source: .*/d' "$file" 17 | done 18 | 19 | cp -r "$tmp"/airflow-operator/*/* deploy/manifests/ 20 | 21 | rm -rf "$tmp" 22 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | 30 | shmVolume: 31 | chmod: 32 | enabled: false 33 | 34 | auth: 35 | username: airflow 36 | password: airflow 37 | database: airflow 38 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | 30 | shmVolume: 31 | chmod: 32 | enabled: false 33 | 34 | auth: 35 | username: airflow 36 | password: airflow 37 | database: airflow 38 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | 30 | shmVolume: 31 | chmod: 32 | enabled: false 33 | 34 | auth: 35 | username: airflow 36 | password: airflow 37 | database: airflow 38 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | 30 | shmVolume: 31 | chmod: 32 | enabled: false 33 | 34 | auth: 35 | username: airflow 36 | password: airflow 37 | database: airflow 38 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | 30 | shmVolume: 31 | chmod: 32 | enabled: false 33 | 34 | auth: 35 | username: airflow 36 | password: airflow 37 | database: airflow 38 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | 30 | shmVolume: 31 | chmod: 32 | enabled: false 33 | 34 | auth: 35 | username: airflow 36 | password: airflow 37 | database: airflow 38 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/50-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | imagePullPolicy: IfNotPresent 22 | stdin: true 23 | tty: true 24 | resources: 25 | requests: 26 | memory: "128Mi" 27 | cpu: "100m" 28 | limits: 29 | memory: "128Mi" 30 | cpu: "400m" 31 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/50-install-airflow-python.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-airflow-python 6 | labels: 7 | app: test-airflow-python 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-airflow-python 13 | template: 14 | metadata: 15 | labels: 16 | app: test-airflow-python 17 | spec: 18 | containers: 19 | - name: test-airflow-python 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | imagePullPolicy: IfNotPresent 22 | stdin: true 23 | tty: true 24 | resources: 25 | requests: 26 | memory: "128Mi" 27 | cpu: "100m" 28 | limits: 29 | memory: "128Mi" 30 | cpu: "400m" 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/01-normal-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Normal issue 3 | about: This is just a normal empty issue with a simple checklist 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Issue checklist 11 | 12 | This is a simple checklist of things to bear in mind when creating a new issue. 13 | 14 | - [ ] **Describe the use-case**: As far as possible, use the pattern "As a [type of user], I would like [feature/functionality] to be able to do [specific action]." This helps identify the feature and the problem it addresses. 15 | - [ ] **Indicate importance and urgency**: Use a scale (e.g., low, medium, high) to indicate the level of importance and urgency. 16 | - [ ] **Work-around**: If there is a known work-around, describe it briefly. 17 | - [ ] **Environment**: Describe the environment where the issue occurs (e.g., SDP version, K8S version, etc.). 18 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/getting_started/code/airflow.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | image: 8 | productVersion: 3.0.6 9 | pullPolicy: IfNotPresent 10 | clusterConfig: 11 | loadExamples: true 12 | exposeConfig: false 13 | credentialsSecret: simple-airflow-credentials 14 | webservers: 15 | roleConfig: 16 | listenerClass: external-unstable 17 | roleGroups: 18 | default: 19 | replicas: 1 20 | celeryExecutors: 21 | roleGroups: 22 | default: 23 | replicas: 1 24 | schedulers: 25 | roleGroups: 26 | default: 27 | replicas: 1 28 | dagProcessors: 29 | roleGroups: 30 | default: 31 | replicas: 1 32 | triggerers: 33 | roleGroups: 34 | default: 35 | replicas: 1 36 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/30-install-keycloak.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: | 6 | INSTANCE_NAME=keycloak1 \ 7 | REALM=test1 \ 8 | USERNAME=jane.doe \ 9 | FIRST_NAME=Jane \ 10 | LAST_NAME=Doe \ 11 | EMAIL=jane.doe@stackable.tech \ 12 | PASSWORD=T8mn72D9 \ 13 | CLIENT_ID=airflow1 \ 14 | CLIENT_SECRET=R1bxHUD569vHeQdw \ 15 | envsubst < install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - 16 | 17 | INSTANCE_NAME=keycloak2 \ 18 | REALM=test2 \ 19 | USERNAME=richard.roe \ 20 | FIRST_NAME=Richard \ 21 | LAST_NAME=Roe \ 22 | EMAIL=richard.roe@stackable.tech \ 23 | PASSWORD=NvfpU518 \ 24 | CLIENT_ID=airflow2 \ 25 | CLIENT_SECRET=scWzh0D4v0GN8NrN \ 26 | envsubst < install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - 27 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-logging.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | image: 8 | productVersion: 3.0.6 9 | clusterConfig: {} 10 | webservers: 11 | roleConfig: 12 | listenerClass: external-unstable 13 | envOverrides: &envOverrides 14 | AIRFLOW__LOGGING__REMOTE_LOGGING: "True" 15 | AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3:///airflow-task-logs/ 16 | # The name of the S3 connection created in the Airflow Web UI 17 | AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | schedulers: 22 | envOverrides: *envOverrides 23 | roleGroups: 24 | default: 25 | replicas: 1 26 | kubernetesExecutors: 27 | envOverrides: *envOverrides 28 | -------------------------------------------------------------------------------- /.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # All defaults or options can be checked here: 3 | # https://github.com/DavidAnson/markdownlint/blob/main/schema/.markdownlint.yaml 4 | 5 | # Default state for all rules 6 | default: true 7 | 8 | # MD013/line-length - Line length 9 | MD013: 10 | # Number of characters 11 | line_length: 9999 12 | # Number of characters for headings 13 | heading_line_length: 9999 14 | # Number of characters for code blocks 15 | code_block_line_length: 9999 16 | 17 | # MD033/no-inline-html 18 | MD033: 19 | allowed_elements: [h1, img, p] 20 | 21 | # MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the same content 22 | MD024: 23 | # Only check sibling headings 24 | siblings_only: true 25 | 26 | # MD041/first-line-heading/first-line-h1 First line in a file should be a top-level heading 27 | MD041: false # Github issues and PRs already have titles, and H1 is enormous in the description box. 28 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/listenerclass.adoc: -------------------------------------------------------------------------------- 1 | = Service exposition with ListenerClasses 2 | :description: Configure Airflow service exposure with ListenerClasses: cluster-internal, external-unstable, or external-stable. 3 | 4 | The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Webserver pod. 5 | The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.webservers.roleConfig.listenerClass`: 6 | 7 | [source,yaml] 8 | ---- 9 | spec: 10 | webservers: 11 | roleConfig: 12 | listenerClass: external-unstable # <1> 13 | config: 14 | ... 15 | schedulers: 16 | ... 17 | celeryExecutors: 18 | ... 19 | ---- 20 | <1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`) at role-level. 21 | This can be set only for the webservers role. 22 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/operations/pod-disruptions.adoc: -------------------------------------------------------------------------------- 1 | = Allowed Pod disruptions 2 | 3 | You can configure the permitted Pod disruptions for Airflow nodes as described in xref:concepts:operations/pod_disruptions.adoc[]. 4 | 5 | Unless you configure something else or disable the default PodDisruptionBudgets (PDBs), the operator writes the following PDBs: 6 | 7 | == Schedulers 8 | Allow only a single scheduler to be offline at any given time, regardless of the number of replicas or `roleGroups`. 9 | 10 | == Webservers 11 | Allow only a single webserver to be offline at any given time, regardless of the number of replicas or `roleGroups`. 12 | 13 | == Executors 14 | * In the case of Celery executors, allow only a single executor to be offline at any given time, regardless of the number of replicas or `roleGroups`. 15 | * In the case of Kubernetes executors, don't deploy any PDB, as it's Airflows responsibility to take care of the executor Pods. 16 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | resources: 30 | requests: 31 | memory: "128Mi" 32 | cpu: "512m" 33 | limits: 34 | memory: "128Mi" 35 | cpu: "1" 36 | 37 | shmVolume: 38 | chmod: 39 | enabled: false 40 | 41 | auth: 42 | username: airflow 43 | password: airflow 44 | database: airflow 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/11_helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | resources: 30 | requests: 31 | memory: "128Mi" 32 | cpu: "512m" 33 | limits: 34 | memory: "128Mi" 35 | cpu: "1" 36 | 37 | shmVolume: 38 | chmod: 39 | enabled: false 40 | 41 | auth: 42 | username: airflow 43 | password: airflow 44 | database: airflow 45 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | resources: 30 | requests: 31 | memory: "128Mi" 32 | cpu: "100m" 33 | limits: 34 | memory: "128Mi" 35 | cpu: "400m" 36 | shmVolume: 37 | chmod: 38 | enabled: false 39 | 40 | auth: 41 | username: airflow 42 | password: airflow 43 | database: airflow 44 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | resources: 30 | requests: 31 | memory: "128Mi" 32 | cpu: "100m" 33 | limits: 34 | memory: "128Mi" 35 | cpu: "400m" 36 | shmVolume: 37 | chmod: 38 | enabled: false 39 | 40 | auth: 41 | username: airflow 42 | password: airflow 43 | database: airflow 44 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | resources: 30 | requests: 31 | memory: "128Mi" 32 | cpu: "100m" 33 | limits: 34 | memory: "128Mi" 35 | cpu: "400m" 36 | shmVolume: 37 | chmod: 38 | enabled: false 39 | 40 | auth: 41 | username: airflow 42 | password: airflow 43 | database: airflow 44 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | podSecurityContext: 22 | {% if test_scenario['values']['openshift'] == 'true' %} 23 | enabled: false 24 | {% else %} 25 | enabled: true 26 | {% endif %} 27 | containerSecurityContext: 28 | enabled: false 29 | resources: 30 | requests: 31 | memory: "128Mi" 32 | cpu: "100m" 33 | limits: 34 | memory: "128Mi" 35 | cpu: "400m" 36 | shmVolume: 37 | chmod: 38 | enabled: false 39 | 40 | auth: 41 | username: airflow 42 | password: airflow 43 | database: airflow 44 | -------------------------------------------------------------------------------- /rust/operator-binary/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "stackable-airflow-operator" 3 | description = "Stackable Operator for Apache Airflow" 4 | version.workspace = true 5 | authors.workspace = true 6 | license.workspace = true 7 | edition.workspace = true 8 | repository.workspace = true 9 | publish = false 10 | 11 | [dependencies] 12 | product-config.workspace = true 13 | stackable-operator.workspace = true 14 | 15 | anyhow.workspace = true 16 | base64.workspace = true 17 | clap.workspace = true 18 | const_format.workspace = true 19 | fnv.workspace = true 20 | futures.workspace = true 21 | indoc.workspace = true 22 | rand.workspace = true 23 | serde.workspace = true 24 | serde_json.workspace = true 25 | serde_yaml.workspace = true 26 | snafu.workspace = true 27 | strum.workspace = true 28 | tokio.workspace = true 29 | tracing.workspace = true 30 | 31 | [build-dependencies] 32 | built.workspace = true 33 | 34 | [dev-dependencies] 35 | rstest.workspace = true 36 | serde_yaml.workspace = true 37 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Helm Chart for Stackable Operator for Apache Airflow 3 | 4 | This Helm Chart can be used to install Custom Resource Definitions and the Operator for Apache Airflow provided by Stackable. 5 | 6 | ## Requirements 7 | 8 | - Create a [Kubernetes Cluster](../Readme.md) 9 | - Install [Helm](https://helm.sh/docs/intro/install/) 10 | 11 | ## Install the Stackable Operator for Apache Airflow 12 | 13 | ```bash 14 | # From the root of the operator repository 15 | make compile-chart 16 | 17 | helm install airflow-operator deploy/helm/airflow-operator 18 | ``` 19 | 20 | ## Usage of the CRDs 21 | 22 | The usage of this operator and its CRDs is described in the [documentation](https://docs.stackable.tech/airflow/index.html) 23 | 24 | The operator has example requests included in the [`/examples`](https://github.com/stackabletech/airflow-operator/tree/main/examples) directory. 25 | 26 | ## Links 27 | 28 | 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/52-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: metrics 6 | timeout: 600 7 | commands: 8 | {% if test_scenario['values']['airflow'].find(",") > 0 %} 9 | - script: | 10 | kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group automatic-log-config --airflow-version "{{ test_scenario['values']['airflow'].split(',')[0] }}" 11 | kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group custom-log-config --airflow-version "{{ test_scenario['values']['airflow'].split(',')[0] }}" 12 | {% else %} 13 | - script: | 14 | kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group automatic-log-config --airflow-version "{{ test_scenario['values']['airflow'] }}" 15 | kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group custom-log-config --airflow-version "{{ test_scenario['values']['airflow'] }}" 16 | {% endif %} 17 | 18 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-gitsync.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | image: 8 | productVersion: 3.0.6 9 | clusterConfig: 10 | loadExamples: false 11 | exposeConfig: false 12 | credentialsSecret: test-airflow-credentials # <1> 13 | dagsGitSync: # <2> 14 | - repo: https://github.com/stackabletech/airflow-operator # <3> 15 | branch: "main" # <4> 16 | gitFolder: "tests/templates/kuttl/mount-dags-gitsync/dags" # <5> 17 | depth: 10 # <6> 18 | wait: 20s # <7> 19 | credentialsSecret: git-credentials # <8> 20 | gitSyncConf: # <9> 21 | --rev: HEAD # <10> 22 | # --rev: git-sync-tag # N.B. tag must be covered by "depth" (the number of commits to clone) 23 | # --rev: 39ee3598bd9946a1d958a448c9f7d3774d7a8043 # N.B. commit must be covered by "depth" 24 | --git-config: http.sslCAInfo:/tmp/ca-cert/ca.crt # <11> 25 | webservers: 26 | ... 27 | -------------------------------------------------------------------------------- /deploy/helm/airflow-operator/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | {{ if .Values.serviceAccount.create -}} 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: {{ include "operator.fullname" . }}-serviceaccount 7 | labels: 8 | {{- include "operator.labels" . | nindent 4 }} 9 | {{- with .Values.serviceAccount.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | # This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. 16 | kind: ClusterRoleBinding 17 | metadata: 18 | name: {{ include "operator.fullname" . }}-clusterrolebinding 19 | labels: 20 | {{- include "operator.labels" . | nindent 4 }} 21 | subjects: 22 | - kind: ServiceAccount 23 | name: {{ include "operator.fullname" . }}-serviceaccount 24 | namespace: {{ .Release.Namespace }} 25 | roleRef: 26 | kind: ClusterRole 27 | name: {{ include "operator.fullname" . }}-clusterrole 28 | apiGroup: rbac.authorization.k8s.io 29 | {{- end }} 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/95-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-log-endpoint 6 | timeout: 240 7 | commands: 8 | {% if test_scenario['values']['executor'] == 'celery' %} 9 | - script: | 10 | set -eu 11 | 12 | # Log-Endpoint Test: 13 | # This is executed from the Webserver as JWT keys must be present. 14 | # A small server is started on each worker that serves the logs on its 15 | # 8793 port for the Webserver: we don't use the token as that is an 16 | # internal implementation, but check that the endpoint is reachable, 17 | # indicated by a 403. 18 | CURL_RESPONSE=$( 19 | kubectl -n $NAMESPACE exec airflow-webserver-default-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-default-headless:8793 2>/dev/null || true);echo "$CODE"' 20 | ) 21 | 22 | # Log-Endpoint Test Assertion: 23 | echo "The HTTP Code is $CURL_RESPONSE (an internal JWT token is needed for full access)" 24 | [ "$CURL_RESPONSE" -eq 403 ] 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/80-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-log-endpoint 6 | timeout: 240 7 | commands: 8 | {% if test_scenario['values']['executor'] == 'celery' %} 9 | - script: | 10 | set -eu 11 | 12 | # Log-Endpoint Test: 13 | # This is executed from the Webserver as JWT keys must be present. 14 | # A small server is started on each worker that serves the logs on its 15 | # 8793 port for the Webserver: we don't use the token as that is an 16 | # internal implementation, but check that the endpoint is reachable, 17 | # indicated by a 403. 18 | CURL_RESPONSE=$( 19 | kubectl -n $NAMESPACE exec airflow-webserver-default-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-default-headless:8793 2>/dev/null || true);echo "$CODE"' 20 | ) 21 | 22 | # Log-Endpoint Test Assertion: 23 | echo "The HTTP Code is $CURL_RESPONSE (an internal JWT token is needed for full access)" 24 | [ "$CURL_RESPONSE" -eq 403 ] 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/70-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-log-endpoint 6 | timeout: 240 7 | commands: 8 | {% if test_scenario['values']['executor'] == 'celery' %} 9 | - script: | 10 | set -eu 11 | 12 | # Log-Endpoint Test: 13 | # This is executed from the Webserver as JWT keys must be present. 14 | # A small server is started on each worker that serves the logs on its 15 | # 8793 port for the Webserver: we don't use the token as that is an 16 | # internal implementation, but check that the endpoint is reachable, 17 | # indicated by a 403. 18 | CURL_RESPONSE=$( 19 | kubectl -n $NAMESPACE exec airflow-webserver-default-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-default-headless:8793 2>/dev/null || true);echo "$CODE"' 20 | ) 21 | 22 | # Log-Endpoint Test Assertion: 23 | echo "The HTTP Code is $CURL_RESPONSE (an internal JWT token is needed for full access)" 24 | [ "$CURL_RESPONSE" -eq 403 ] 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/50-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-log-endpoint 6 | timeout: 240 7 | commands: 8 | {% if test_scenario['values']['executor'] == 'celery' %} 9 | - script: | 10 | set -eu 11 | 12 | # Log-Endpoint Test: 13 | # This is executed from the Webserver as JWT keys must be present. 14 | # A small server is started on each worker that serves the logs on its 15 | # 8793 port for the Webserver: we don't use the token as that is an 16 | # internal implementation, but check that the endpoint is reachable, 17 | # indicated by a 403. 18 | CURL_RESPONSE=$( 19 | kubectl -n $NAMESPACE exec airflow-webserver-default-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-default-headless:8793 2>/dev/null || true);echo "$CODE"' 20 | ) 21 | 22 | # Log-Endpoint Test Assertion: 23 | echo "The HTTP Code is $CURL_RESPONSE (an internal JWT token is needed for full access)" 24 | [ "$CURL_RESPONSE" -eq 403 ] 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/create_ldap_user.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # To check the existing users 4 | # ldapsearch -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w admin -b ou=users,dc=example,dc=org 5 | 6 | # To check the new user 7 | # ldapsearch -H ldap://localhost:1389 -D cn=integrationtest,ou=users,dc=example,dc=org -w integrationtest -b ou=users,dc=example,dc=org 8 | 9 | cat << 'EOF' | ldapadd -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w admin 10 | dn: cn=integrationtest,ou=users,dc=example,dc=org 11 | objectClass: inetOrgPerson 12 | objectClass: posixAccount 13 | objectClass: shadowAccount 14 | cn: integrationtest 15 | uid: integrationtest 16 | givenName: Stackable 17 | sn: Integration-Test 18 | mail: integrationtest@stackable.de 19 | uidNumber: 16842 20 | gidNumber: 100 21 | homeDirectory: /home/integrationtest 22 | loginShell: /bin/bash 23 | userPassword: {crypt}x 24 | shadowLastChange: 0 25 | shadowMax: 0 26 | shadowWarning: 0 27 | EOF 28 | 29 | ldappasswd -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w admin -s integrationtest "cn=integrationtest,ou=users,dc=example,dc=org" 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/70-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-log-endpoint 6 | timeout: 240 7 | commands: 8 | {% if test_scenario['values']['executor'] == 'celery' %} 9 | - script: | 10 | set -eu 11 | 12 | # Log-Endpoint Test: 13 | # This is executed from the Webserver as JWT keys must be present. 14 | # A small server is started on each worker that serves the logs on its 15 | # 8793 port for the Webserver: we don't use the token as that is an 16 | # internal implementation, but check that the endpoint is reachable, 17 | # indicated by a 403. 18 | CURL_RESPONSE=$( 19 | kubectl -n $NAMESPACE exec airflow-webserver-default-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-default-headless:8793 2>/dev/null || true);echo "$CODE"' 20 | ) 21 | 22 | # Log-Endpoint Test Assertion: 23 | echo "The HTTP Code is $CURL_RESPONSE (an internal JWT token is needed for full access)" 24 | [ "$CURL_RESPONSE" -eq 403 ] 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/70-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-log-endpoint 6 | timeout: 240 7 | commands: 8 | {% if test_scenario['values']['executor'] == 'celery' %} 9 | - script: | 10 | set -eu 11 | 12 | # Log-Endpoint Test: 13 | # This is executed from the Webserver as JWT keys must be present. 14 | # A small server is started on each worker that serves the logs on its 15 | # 8793 port for the Webserver: we don't use the token as that is an 16 | # internal implementation, but check that the endpoint is reachable, 17 | # indicated by a 403. 18 | CURL_RESPONSE=$( 19 | kubectl -n $NAMESPACE exec airflow-webserver-default-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-default-headless:8793 2>/dev/null || true);echo "$CODE"' 20 | ) 21 | 22 | # Log-Endpoint Test Assertion: 23 | echo "The HTTP Code is $CURL_RESPONSE (an internal JWT token is needed for full access)" 24 | [ "$CURL_RESPONSE" -eq 403 ] 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /scripts/docs_templating.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Reads a file with variables to insert into templates, and templates all .*.j2 files 5 | # in the 'docs' directory. 6 | # 7 | # dependencies 8 | # pip install jinja2-cli 9 | 10 | docs_dir="$(dirname "$0")/../docs" 11 | templating_vars_file="$docs_dir/templating_vars.yaml" 12 | 13 | # Check if files need templating 14 | if [[ -z $(find "$docs_dir" -name '*.j2') ]]; 15 | then 16 | echo "No files need templating, exiting." 17 | exit 18 | fi 19 | 20 | # Check if jinja2 is there 21 | if ! command -v jinja2 &> /dev/null 22 | then 23 | echo "jinja2 could not be found. Use 'pip install jinja2-cli' to install it." 24 | exit 1 25 | fi 26 | 27 | # Check if templating vars file exists 28 | if [[ ! -f "$templating_vars_file" ]]; 29 | then 30 | echo "$templating_vars_file does not exist, cannot start templating." 31 | fi 32 | 33 | find "$docs_dir" -name '*.j2' | 34 | while read -r file 35 | do 36 | new_file_name=${file%.j2} # Remove .j2 suffix 37 | echo "templating $new_file_name" 38 | jinja2 "$file" "$templating_vars_file" -o "$new_file_name" 39 | done 40 | 41 | echo "done" 42 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/getting_started/index.adoc: -------------------------------------------------------------------------------- 1 | = Getting started 2 | :description: Get started with the Stackable Operator for Apache Airflow by installing the operator, SQL database, and Redis, then setting up and running your first DAG. 3 | 4 | This guide gets you started with Airflow using the Stackable Operator. 5 | It guides you through the installation of the Operator as well as an SQL database and Redis instance for trial usage, setting up your first Airflow cluster and connecting to it, and viewing and running one of the example workflows (called DAGs = Direct Acyclic Graphs). 6 | 7 | == Prerequisites for this guide 8 | 9 | You need: 10 | 11 | * a Kubernetes cluster 12 | * kubectl 13 | * Helm 14 | 15 | Resource sizing depends on cluster type(s), usage and scope, but as a minimum starting point the following resources are recommended for this operator: 16 | 17 | include::partial$hardware-requirements.adoc[] 18 | 19 | == What's next 20 | 21 | The Guide is divided into two steps: 22 | 23 | * xref:getting_started/installation.adoc[Installing the Operators]. 24 | * xref:getting_started/first_steps.adoc[Setting up the Airflow cluster and running an example DAG]. 25 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-dags-configmap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | image: 8 | productVersion: 3.0.6 9 | clusterConfig: 10 | loadExamples: false 11 | exposeConfig: false 12 | credentialsSecret: simple-airflow-credentials 13 | volumes: 14 | - name: cm-dag # <3> 15 | configMap: 16 | name: cm-dag # <4> 17 | volumeMounts: 18 | - name: cm-dag # <5> 19 | mountPath: /dags/test_airflow_dag.py # <6> 20 | subPath: test_airflow_dag.py # <7> 21 | webservers: 22 | roleConfig: 23 | listenerClass: external-unstable 24 | roleGroups: 25 | default: 26 | envOverrides: 27 | AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8> 28 | replicas: 1 29 | celeryExecutors: 30 | roleGroups: 31 | default: 32 | envOverrides: 33 | AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8> 34 | replicas: 2 35 | schedulers: 36 | roleGroups: 37 | default: 38 | envOverrides: 39 | AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8> 40 | replicas: 1 41 | -------------------------------------------------------------------------------- /docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-xcom.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: airflow.stackable.tech/v1alpha1 3 | kind: AirflowCluster 4 | metadata: 5 | name: airflow 6 | spec: 7 | image: 8 | productVersion: 3.0.6 9 | clusterConfig: {} 10 | webservers: 11 | roleConfig: 12 | listenerClass: external-unstable 13 | envOverrides: &envOverrides 14 | AIRFLOW__CORE__XCOM_BACKEND: airflow.providers.common.io.xcom.backend.XComObjectStorageBackend 15 | # The connection id is obtained from the user part of the url that you will provide 16 | AIRFLOW__COMMON_IO__XCOM_OBJECTSTORAGE_PATH: s3://minio@/airflow-xcom 17 | # Any object smaller than the threshold in bytes will be stored in the database and anything larger will be be put in object storage 18 | AIRFLOW__COMMON_IO__XCOM_OBJECTSTORAGE_THRESHOLD: "1024" # 1KiB 19 | AIRFLOW__COMMON_IO__XCOM_OBJECTSTORAGE_COMPRESSION: gzip 20 | roleGroups: 21 | default: 22 | replicas: 1 23 | schedulers: 24 | envOverrides: *envOverrides 25 | roleGroups: 26 | default: 27 | replicas: 1 28 | kubernetesExecutors: 29 | envOverrides: *envOverrides 30 | -------------------------------------------------------------------------------- /tests/kuttl-test.yaml.jinja2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestSuite 4 | testDirs: 5 | {% for testcase in testinput.tests %} 6 | - ./tests/{{ testcase.name }} 7 | {% endfor %} 8 | 9 | startKIND: false 10 | suppress: ["events"] 11 | parallel: 2 12 | 13 | # The timeout (in seconds) is used when namespaces are created or 14 | # deleted, and, if not overridden, in TestSteps, TestAsserts, and 15 | # Commands. If not set, the timeout is 30 seconds by default. 16 | # 17 | # The deletion of a namespace can take a while until all resources, 18 | # especially PersistentVolumeClaims, are gracefully shut down. If the 19 | # timeout is reached in the meantime, even a successful test case is 20 | # considered a failure. 21 | # 22 | # For instance, the termination grace period of the Vector aggregator in 23 | # the logging tests is set to 60 seconds. If there are logs entries 24 | # which could not be forwarded yet to the external aggregator defined in 25 | # the VECTOR_AGGREGATOR environment variable, then the test aggregator 26 | # uses this period of time by trying to forward the events. In this 27 | # case, deleting a namespace with several Pods takes about 90 seconds. 28 | timeout: 300 29 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/new_version.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: New Version 3 | about: Request support for a new product version 4 | title: "[NEW VERSION]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Which new version of Apache Airflow should we support? 11 | 12 | Please specify the version, version range or version numbers to support, please also add these to the issue title 13 | 14 | ## Additional information 15 | 16 | If possible, provide a link to release notes/changelog 17 | 18 | ## Changes required 19 | 20 | Are there any upstream changes that we need to support? 21 | e.g. new features, changed features, deprecated features etc. 22 | 23 | ## Implementation checklist 24 | 25 | 29 | 30 | - [ ] Update the Docker image 31 | - [ ] Update documentation to include supported version(s) 32 | - [ ] Update and test getting started guide with updated version(s) 33 | - [ ] Update operator to support the new version (if needed) 34 | - [ ] Update integration tests to test use the new versions (in addition or replacing old versions 35 | - [ ] Update examples to use new versions 36 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pre-release-getting-started-script.md: -------------------------------------------------------------------------------- 1 | ## Check and Update Getting Started Script 2 | 3 | 7 | 8 | 11 | 12 | Part of 13 | 14 | > [!NOTE] 15 | > During a Stackable release we need to check (and optionally update) the 16 | > getting-started scripts to ensure they still work after product and operator 17 | > updates. 18 | 19 | ```shell 20 | # Some of the scripts are in a code/ subdirectory 21 | # pushd docs/modules/superset/examples/getting_started 22 | # pushd docs/modules/superset/examples/getting_started/code 23 | pushd $(fd -td getting_started | grep examples); cd code 2>/dev/null || true 24 | 25 | # Make a fresh cluster (~12 seconds) 26 | kind delete cluster && kind create cluster 27 | ./getting_started.sh stackablectl 28 | 29 | # Make a fresh cluster (~12 seconds) 30 | kind delete cluster && kind create cluster 31 | ./getting_started.sh helm 32 | 33 | popd 34 | ``` 35 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | replica: 38 | replicaCount: 1 39 | podSecurityContext: 40 | {% if test_scenario['values']['openshift'] == 'true' %} 41 | enabled: false 42 | {% else %} 43 | enabled: true 44 | {% endif %} 45 | containerSecurityContext: 46 | enabled: false 47 | 48 | auth: 49 | password: redis 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | replica: 38 | replicaCount: 1 39 | podSecurityContext: 40 | {% if test_scenario['values']['openshift'] == 'true' %} 41 | enabled: false 42 | {% else %} 43 | enabled: true 44 | {% endif %} 45 | containerSecurityContext: 46 | enabled: false 47 | 48 | auth: 49 | password: redis 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | replica: 38 | replicaCount: 1 39 | podSecurityContext: 40 | {% if test_scenario['values']['openshift'] == 'true' %} 41 | enabled: false 42 | {% else %} 43 | enabled: true 44 | {% endif %} 45 | containerSecurityContext: 46 | enabled: false 47 | 48 | auth: 49 | password: redis 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/triggerer/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | replica: 38 | replicaCount: 1 39 | podSecurityContext: 40 | {% if test_scenario['values']['openshift'] == 'true' %} 41 | enabled: false 42 | {% else %} 43 | enabled: true 44 | {% endif %} 45 | containerSecurityContext: 46 | enabled: false 47 | 48 | auth: 49 | password: redis 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | replica: 38 | replicaCount: 1 39 | podSecurityContext: 40 | {% if test_scenario['values']['openshift'] == 'true' %} 41 | enabled: false 42 | {% else %} 43 | enabled: true 44 | {% endif %} 45 | containerSecurityContext: 46 | enabled: false 47 | 48 | auth: 49 | password: redis 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/opa/20-install-opa.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: install-opa 6 | --- 7 | apiVersion: opa.stackable.tech/v1alpha1 8 | kind: OpaCluster 9 | metadata: 10 | # The OpaCluster is intentionally not only called "opa" to ensure that our custom OPA URL is 11 | # used and not some default value of "opa". 12 | name: test-opa 13 | spec: 14 | image: 15 | {% if test_scenario['values']['opa-latest'].find(",") > 0 %} 16 | custom: "{{ test_scenario['values']['opa-latest'].split(',')[1] }}" 17 | productVersion: "{{ test_scenario['values']['opa-latest'].split(',')[0] }}" 18 | {% else %} 19 | productVersion: "{{ test_scenario['values']['opa-latest'] }}" 20 | {% endif %} 21 | pullPolicy: IfNotPresent 22 | clusterConfig: 23 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 24 | vectorAggregatorConfigMapName: vector-aggregator-discovery 25 | {% endif %} 26 | servers: 27 | config: 28 | logging: 29 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 30 | containers: 31 | opa: 32 | loggers: 33 | decision: 34 | level: INFO 35 | roleGroups: 36 | default: {} 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-configmap/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | replica: 38 | replicaCount: 1 39 | podSecurityContext: 40 | {% if test_scenario['values']['openshift'] == 'true' %} 41 | enabled: false 42 | {% else %} 43 | enabled: true 44 | {% endif %} 45 | containerSecurityContext: 46 | enabled: false 47 | 48 | auth: 49 | password: redis 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/mount-dags-gitsync/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | replica: 38 | replicaCount: 1 39 | podSecurityContext: 40 | {% if test_scenario['values']['openshift'] == 'true' %} 41 | enabled: false 42 | {% else %} 43 | enabled: true 44 | {% endif %} 45 | containerSecurityContext: 46 | enabled: false 47 | 48 | auth: 49 | password: redis 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned-resources/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | 37 | replica: 38 | replicaCount: 1 39 | podSecurityContext: 40 | {% if test_scenario['values']['openshift'] == 'true' %} 41 | enabled: false 42 | {% else %} 43 | enabled: true 44 | {% endif %} 45 | containerSecurityContext: 46 | enabled: false 47 | 48 | auth: 49 | password: redis 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/41-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-airflow-cluster 6 | timeout: 1200 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: airflow-webserver-automatic-log-config 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | --- 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: airflow-webserver-custom-log-config 20 | status: 21 | readyReplicas: 1 22 | replicas: 1 23 | {% if test_scenario['values']['executor'] == 'celery' %} 24 | --- 25 | apiVersion: apps/v1 26 | kind: StatefulSet 27 | metadata: 28 | name: airflow-worker-automatic-log-config 29 | status: 30 | readyReplicas: 1 31 | replicas: 1 32 | --- 33 | apiVersion: apps/v1 34 | kind: StatefulSet 35 | metadata: 36 | name: airflow-worker-custom-log-config 37 | status: 38 | readyReplicas: 1 39 | replicas: 1 40 | {% endif %} 41 | --- 42 | apiVersion: apps/v1 43 | kind: StatefulSet 44 | metadata: 45 | name: airflow-scheduler-automatic-log-config 46 | status: 47 | readyReplicas: 1 48 | replicas: 1 49 | --- 50 | apiVersion: apps/v1 51 | kind: StatefulSet 52 | metadata: 53 | name: airflow-scheduler-custom-log-config 54 | status: 55 | readyReplicas: 1 56 | replicas: 1 57 | -------------------------------------------------------------------------------- /examples/simple-airflow-cluster.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: simple-airflow-credentials 6 | type: Opaque 7 | stringData: 8 | adminUser.username: airflow 9 | adminUser.firstname: Airflow 10 | adminUser.lastname: Admin 11 | adminUser.email: airflow@airflow.com 12 | adminUser.password: airflow 13 | connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 14 | # Only needed when using celery workers (instead of Kubernetes executors) 15 | connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow 16 | connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 17 | --- 18 | apiVersion: airflow.stackable.tech/v1alpha1 19 | kind: AirflowCluster 20 | metadata: 21 | name: airflow 22 | spec: 23 | image: 24 | productVersion: 3.0.6 25 | clusterConfig: 26 | loadExamples: true 27 | exposeConfig: false 28 | credentialsSecret: simple-airflow-credentials 29 | webservers: 30 | roleConfig: 31 | listenerClass: external-unstable 32 | roleGroups: 33 | default: 34 | replicas: 1 35 | celeryExecutors: 36 | roleGroups: 37 | default: 38 | replicas: 2 39 | schedulers: 40 | roleGroups: 41 | default: 42 | replicas: 1 43 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["rust/operator-binary"] 3 | resolver = "2" 4 | 5 | [workspace.package] 6 | version = "0.0.0-dev" 7 | authors = ["Stackable GmbH "] 8 | license = "OSL-3.0" 9 | edition = "2021" 10 | repository = "https://github.com/stackabletech/airflow-operator" 11 | 12 | [workspace.dependencies] 13 | product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } 14 | stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.101.1" } 15 | 16 | anyhow = "1.0" 17 | base64 = "0.22" 18 | built = { version = "0.8", features = ["chrono", "git2"] } 19 | clap = "4.5" 20 | const_format = "0.2" 21 | fnv = "1.0" 22 | futures = { version = "0.3", features = ["compat"] } 23 | indoc = "2.0" 24 | rand = "0.9.0" 25 | rstest = "0.26" 26 | semver = "1.0" 27 | serde = { version = "1.0", features = ["derive"] } 28 | serde_json = "1.0" 29 | serde_yaml = "0.9" 30 | snafu = "0.8" 31 | strum = { version = "0.27", features = ["derive"] } 32 | tokio = { version = "1.40", features = ["full"] } 33 | tracing = "0.1" 34 | 35 | [patch."https://github.com/stackabletech/operator-rs.git"] 36 | # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } 37 | # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } 38 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/db-init.adoc: -------------------------------------------------------------------------------- 1 | = Database initialization 2 | :description: Configure Airflow Database start-up. 3 | 4 | By default, Airflow will run database initialization routines (checking and/or creating the metadata schema and creating an admin user) on start-up. 5 | These are idempotent and can be run every time as the overhead is minimal. 6 | However, if these steps should be skipped, a running Airflow cluster can be patched with a resource like this to deactivate the initialization: 7 | 8 | [source,yaml] 9 | ---- 10 | --- 11 | apiVersion: airflow.stackable.tech/v1alpha1 12 | kind: AirflowCluster 13 | metadata: 14 | name: airflow 15 | spec: 16 | clusterConfig: 17 | databaseInitialization: 18 | enabled: false # <1> 19 | ---- 20 | <1> Turn off the initialization routine by setting `databaseInitialization.enabled` to `false` 21 | 22 | NOTE: The field `databaseInitialization.enabled` is `true` by default to be backwards-compatible. 23 | A fresh Airflow cluster cannot be created with this field set to `false` as this results in missing metadata in the Airflow database. 24 | 25 | WARNING: Setting `databaseInitialization.enabled` to `false` is an unsupported operation as subsequent updates to a running Airflow cluster can result in broken behaviour due to inconsistent metadata. 26 | Only set `databaseInitialization.enabled` to `false` if you know what you are doing! 27 | -------------------------------------------------------------------------------- /.github/workflows/pr_pre-commit.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: pre-commit 3 | 4 | on: 5 | pull_request: 6 | merge_group: 7 | 8 | env: 9 | CARGO_TERM_COLOR: always 10 | NIX_PKG_MANAGER_VERSION: "2.30.0" 11 | RUST_TOOLCHAIN_VERSION: "nightly-2025-10-23" 12 | HADOLINT_VERSION: "v2.14.0" 13 | PYTHON_VERSION: "3.14" 14 | JINJA2_CLI_VERSION: "0.8.2" 15 | 16 | jobs: 17 | pre-commit: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Install host dependencies 21 | uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 22 | with: 23 | packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https 24 | version: ubuntu-latest 25 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 26 | with: 27 | persist-credentials: false 28 | submodules: recursive 29 | fetch-depth: 0 30 | - uses: stackabletech/actions/run-pre-commit@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 31 | with: 32 | python-version: ${{ env.PYTHON_VERSION }} 33 | rust: ${{ env.RUST_TOOLCHAIN_VERSION }} 34 | hadolint: ${{ env.HADOLINT_VERSION }} 35 | nix: ${{ env.NIX_PKG_MANAGER_VERSION }} 36 | nix-github-token: ${{ secrets.GITHUB_TOKEN }} 37 | jinja2-cli: ${{ env.JINJA2_CLI_VERSION }} 38 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/prepared-logs.py.json: -------------------------------------------------------------------------------- 1 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "DEBUG", "name": "TestLogger", "message": "Valid log event with log level DEBUG"} 2 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "INFO", "name": "TestLogger", "message": "Valid log event with log level INFO"} 3 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "WARNING", "name": "TestLogger", "message": "Valid log event with log level WARNING"} 4 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "ERROR", "name": "TestLogger", "message": "Valid log event with log level ERROR"} 5 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "CRITICAL", "name": "TestLogger", "message": "Valid log event with log level CRITICAL"} 6 | {"levelname": "INFO", "name": "TestLogger", "message": "Invalid log event without a timestamp"} 7 | {"asctime": "unparsable timestamp", "levelname": "INFO", "name": "TestLogger", "message": "Invalid log event with an unparsable timestamp"} 8 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "INFO", "message": "Invalid log event without a logger"} 9 | {"asctime": "2024-01-01 00:00:00,000", "name": "TestLogger", "message": "Invalid log event without a level"} 10 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "FATAL", "name": "TestLogger", "message": "Invalid log event with an unknown level"} 11 | {"asctime": "2024-01-01 00:00:00,000", "levelname": "INFO", "name": "TestLogger"} 12 | "true" 13 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pre-release-rust-deps.md: -------------------------------------------------------------------------------- 1 | ## Bump Rust Dependencies for Stackable Release YY.M.X 2 | 3 | 7 | 8 | 11 | 12 | Part of 13 | 14 | > [!NOTE] 15 | > During a Stackable release we need to update various Rust dependencies before 16 | > entering the final release period to ensure we run the latest versions of 17 | > crates. These bumps also include previously updated and released crates from 18 | > the `operator-rs` repository. 19 | 20 | ### Tasks 21 | 22 | - [ ] Bump Rust Dependencies, see below for more details. 23 | - [ ] Add changelog entry stating which important crates were bumped (including the version). 24 | 25 | > [!NOTE] 26 | > The bumping / updating of Rust dependencies is done in multiple steps: 27 | > 28 | > 1. Update the minimum Version in the root `Cargo.toml` manifest. 29 | > 2. Run the `cargo update` command, which also updates the `Cargo.lock` file. 30 | > 3. Lastly, run `make regenerate-nix` to update the `Cargo.nix` file. 31 | 32 | ### Bump Rust Dependencies 33 | 34 | - [ ] Bump `stackable-operator` and friends 35 | - [ ] Bump `product-config` 36 | - [ ] Bump all other dependencies 37 | -------------------------------------------------------------------------------- /docs/modules/airflow/partials/nav.adoc: -------------------------------------------------------------------------------- 1 | * xref:airflow:getting_started/index.adoc[] 2 | ** xref:airflow:getting_started/installation.adoc[] 3 | ** xref:airflow:getting_started/first_steps.adoc[] 4 | * xref:airflow:required-external-components.adoc[] 5 | * xref:airflow:usage-guide/index.adoc[] 6 | ** xref:airflow:usage-guide/db-init.adoc[] 7 | ** xref:airflow:usage-guide/mounting-dags.adoc[] 8 | ** xref:airflow:usage-guide/applying-custom-resources.adoc[] 9 | ** xref:airflow:usage-guide/listenerclass.adoc[] 10 | ** xref:airflow:usage-guide/storage-resources.adoc[] 11 | ** xref:airflow:usage-guide/security.adoc[] 12 | ** xref:airflow:usage-guide/logging.adoc[] 13 | ** xref:airflow:usage-guide/monitoring.adoc[] 14 | ** xref:airflow:usage-guide/using-kubernetes-executors.adoc[] 15 | ** xref:airflow:usage-guide/overrides.adoc[] 16 | ** xref:airflow:usage-guide/operations/index.adoc[] 17 | *** xref:airflow:usage-guide/operations/cluster-operations.adoc[] 18 | *** xref:airflow:usage-guide/operations/pod-placement.adoc[] 19 | *** xref:airflow:usage-guide/operations/pod-disruptions.adoc[] 20 | *** xref:airflow:usage-guide/operations/graceful-shutdown.adoc[] 21 | * xref:airflow:troubleshooting/index.adoc[] 22 | * xref:airflow:reference/index.adoc[] 23 | ** xref:airflow:reference/crds.adoc[] 24 | *** {crd-docs}/airflow.stackable.tech/airflowcluster/v1alpha1/[AirflowCluster {external-link-icon}^] 25 | ** xref:airflow:reference/commandline-parameters.adoc[] 26 | ** xref:airflow:reference/environment-variables.adoc[] 27 | -------------------------------------------------------------------------------- /tests/templates/kuttl/overrides/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | resources: 37 | requests: 38 | memory: "128Mi" 39 | cpu: "200m" 40 | limits: 41 | memory: "128Mi" 42 | cpu: "800m" 43 | 44 | replica: 45 | replicaCount: 1 46 | podSecurityContext: 47 | {% if test_scenario['values']['openshift'] == 'true' %} 48 | enabled: false 49 | {% else %} 50 | enabled: true 51 | {% endif %} 52 | containerSecurityContext: 53 | enabled: false 54 | resources: 55 | requests: 56 | memory: "128Mi" 57 | cpu: "100m" 58 | limits: 59 | memory: "128Mi" 60 | cpu: "400m" 61 | 62 | auth: 63 | password: redis 64 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | resources: 37 | requests: 38 | memory: "128Mi" 39 | cpu: "200m" 40 | limits: 41 | memory: "128Mi" 42 | cpu: "800m" 43 | 44 | replica: 45 | replicaCount: 1 46 | podSecurityContext: 47 | {% if test_scenario['values']['openshift'] == 'true' %} 48 | enabled: false 49 | {% else %} 50 | enabled: true 51 | {% endif %} 52 | containerSecurityContext: 53 | enabled: false 54 | resources: 55 | requests: 56 | memory: "128Mi" 57 | cpu: "100m" 58 | limits: 59 | memory: "128Mi" 60 | cpu: "400m" 61 | 62 | auth: 63 | password: redis 64 | -------------------------------------------------------------------------------- /tests/templates/kuttl/remote-logging/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | resources: 37 | requests: 38 | memory: "128Mi" 39 | cpu: "200m" 40 | limits: 41 | memory: "128Mi" 42 | cpu: "800m" 43 | 44 | replica: 45 | replicaCount: 1 46 | podSecurityContext: 47 | {% if test_scenario['values']['openshift'] == 'true' %} 48 | enabled: false 49 | {% else %} 50 | enabled: true 51 | {% endif %} 52 | containerSecurityContext: 53 | enabled: false 54 | resources: 55 | requests: 56 | memory: "128Mi" 57 | cpu: "100m" 58 | limits: 59 | memory: "128Mi" 60 | cpu: "400m" 61 | 62 | auth: 63 | password: redis 64 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/helm-bitnami-redis-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images 5 | image: 6 | repository: bitnamilegacy/redis 7 | sentinel: 8 | image: 9 | repository: bitnamilegacy/redis-sentinel 10 | metrics: 11 | image: 12 | repository: bitnamilegacy/redis-exporter 13 | kubectl: 14 | image: 15 | repository: bitnamilegacy/kubectl 16 | sysctl: 17 | image: 18 | repository: bitnamilegacy/os-shell 19 | 20 | volumePermissions: 21 | enabled: false 22 | image: 23 | repository: bitnamilegacy/os-shell 24 | containerSecurityContext: 25 | runAsUser: auto 26 | 27 | master: 28 | podSecurityContext: 29 | {% if test_scenario['values']['openshift'] == 'true' %} 30 | enabled: false 31 | {% else %} 32 | enabled: true 33 | {% endif %} 34 | containerSecurityContext: 35 | enabled: false 36 | resources: 37 | requests: 38 | memory: "128Mi" 39 | cpu: "200m" 40 | limits: 41 | memory: "128Mi" 42 | cpu: "800m" 43 | 44 | replica: 45 | replicaCount: 1 46 | podSecurityContext: 47 | {% if test_scenario['values']['openshift'] == 'true' %} 48 | enabled: false 49 | {% else %} 50 | enabled: true 51 | {% endif %} 52 | containerSecurityContext: 53 | enabled: false 54 | resources: 55 | requests: 56 | memory: "128Mi" 57 | cpu: "100m" 58 | limits: 59 | memory: "128Mi" 60 | cpu: "400m" 61 | 62 | auth: 63 | password: redis 64 | -------------------------------------------------------------------------------- /rust/operator-binary/src/operations/graceful_shutdown.rs: -------------------------------------------------------------------------------- 1 | use snafu::{ResultExt, Snafu}; 2 | use stackable_operator::builder::pod::PodBuilder; 3 | 4 | use crate::crd::{AirflowConfig, ExecutorConfig}; 5 | 6 | #[derive(Debug, Snafu)] 7 | pub enum Error { 8 | #[snafu(display("Failed to set terminationGracePeriod"))] 9 | SetTerminationGracePeriod { 10 | source: stackable_operator::builder::pod::Error, 11 | }, 12 | } 13 | 14 | pub fn add_airflow_graceful_shutdown_config( 15 | merged_config: &AirflowConfig, 16 | pod_builder: &mut PodBuilder, 17 | ) -> Result<(), Error> { 18 | // This must be always set by the merge mechanism, as we provide a default value, 19 | // users can not disable graceful shutdown. 20 | if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout { 21 | pod_builder 22 | .termination_grace_period(&graceful_shutdown_timeout) 23 | .context(SetTerminationGracePeriodSnafu)?; 24 | } 25 | 26 | Ok(()) 27 | } 28 | 29 | pub fn add_executor_graceful_shutdown_config( 30 | merged_config: &ExecutorConfig, 31 | pod_builder: &mut PodBuilder, 32 | ) -> Result<(), Error> { 33 | // This must be always set by the merge mechanism, as we provide a default value, 34 | // users can not disable graceful shutdown. 35 | if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout { 36 | pod_builder 37 | .termination_grace_period(&graceful_shutdown_timeout) 38 | .context(SetTerminationGracePeriodSnafu)?; 39 | } 40 | 41 | Ok(()) 42 | } 43 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/70-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-log-endpoint 6 | timeout: 240 7 | commands: 8 | {% if test_scenario['values']['executor'] == 'celery' %} 9 | - script: | 10 | set -eu 11 | 12 | # Log-Endpoint Test: 13 | # This is executed from the Webserver as JWT keys must be present. 14 | # A small server is started on each worker that serves the logs on its 15 | # 8793 port for the Webserver: we don't use the token as that is an 16 | # internal implementation, but check that the endpoint is reachable, 17 | # indicated by a 403. 18 | # Rolegroup custom-log-config 19 | CURL_RESPONSE_CUSTOM=$( 20 | kubectl -n $NAMESPACE exec airflow-webserver-custom-log-config-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-custom-log-config-headless:8793 2>/dev/null || true);echo "$CODE"' 21 | ) 22 | 23 | # Log-Endpoint Test Assertions: 24 | echo "The HTTP Code is $CURL_RESPONSE_CUSTOM (an internal JWT token is needed for full access)" 25 | 26 | # Rolegroup automatic-log-config 27 | CURL_RESPONSE_AUTO=$( 28 | kubectl -n $NAMESPACE exec airflow-webserver-automatic-log-config-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-automatic-log-config-headless:8793 2>/dev/null || true);echo "$CODE"' 29 | ) 30 | echo "The HTTP Code is $CURL_RESPONSE_AUTO (an internal JWT token is needed for full access)" 31 | [ "$CURL_RESPONSE_CUSTOM" -eq 403 ] && [ "$CURL_RESPONSE_AUTO" -eq 403 ] 32 | {% endif %} 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/test_log_aggregation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import requests 3 | 4 | 5 | def check_sent_events(): 6 | response = requests.post( 7 | "http://airflow-vector-aggregator:8686/graphql", 8 | json={ 9 | "query": """ 10 | { 11 | transforms(first:100) { 12 | nodes { 13 | componentId 14 | metrics { 15 | sentEventsTotal { 16 | sentEventsTotal 17 | } 18 | } 19 | } 20 | } 21 | } 22 | """ 23 | }, 24 | ) 25 | 26 | assert ( 27 | response.status_code == 200 28 | ), "Cannot access the API of the vector aggregator." 29 | 30 | result = response.json() 31 | 32 | transforms = result["data"]["transforms"]["nodes"] 33 | for transform in transforms: 34 | sentEvents = transform["metrics"]["sentEventsTotal"] 35 | componentId = transform["componentId"] 36 | 37 | if componentId == "filteredInvalidEvents": 38 | assert ( 39 | sentEvents is None or sentEvents["sentEventsTotal"] == 0 40 | ), "Invalid log events were sent." 41 | else: 42 | assert ( 43 | sentEvents is not None and sentEvents["sentEventsTotal"] > 0 44 | ), f'No events were sent in "{componentId}".' 45 | 46 | 47 | if __name__ == "__main__": 48 | check_sent_events() 49 | print("Test successful!") 50 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/20-stop-airflow.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: kuttl.dev/v1beta1 2 | kind: TestStep 3 | metadata: 4 | name: stop-airflow 5 | timeout: 180 6 | --- 7 | apiVersion: airflow.stackable.tech/v1alpha1 8 | kind: AirflowCluster 9 | metadata: 10 | name: airflow 11 | spec: 12 | clusterOperation: 13 | reconciliationPaused: false 14 | stopped: true 15 | image: 16 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 17 | custom: "{{ test_scenario['values']['airflow-latest'].split(',')[1] }}" 18 | productVersion: "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 19 | {% else %} 20 | productVersion: "{{ test_scenario['values']['airflow-latest'] }}" 21 | {% endif %} 22 | pullPolicy: IfNotPresent 23 | clusterConfig: 24 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 25 | vectorAggregatorConfigMapName: vector-aggregator-discovery 26 | {% endif %} 27 | credentialsSecret: test-airflow-credentials 28 | webservers: 29 | roleConfig: 30 | listenerClass: external-unstable 31 | config: 32 | logging: 33 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 34 | roleGroups: 35 | default: 36 | replicas: 1 37 | celeryExecutors: 38 | config: 39 | logging: 40 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 41 | roleGroups: 42 | default: 43 | replicas: 3 # ignored because paused 44 | schedulers: 45 | config: 46 | logging: 47 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 48 | roleGroups: 49 | default: 50 | replicas: 1 51 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster-operation/10-pause-airflow.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: kuttl.dev/v1beta1 2 | kind: TestStep 3 | metadata: 4 | name: pause-airflow 5 | timeout: 480 6 | --- 7 | apiVersion: airflow.stackable.tech/v1alpha1 8 | kind: AirflowCluster 9 | metadata: 10 | name: airflow 11 | spec: 12 | clusterOperation: 13 | reconciliationPaused: true 14 | stopped: false 15 | image: 16 | {% if test_scenario['values']['airflow-latest'].find(",") > 0 %} 17 | custom: "{{ test_scenario['values']['airflow-latest'].split(',')[1] }}" 18 | productVersion: "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" 19 | {% else %} 20 | productVersion: "{{ test_scenario['values']['airflow-latest'] }}" 21 | {% endif %} 22 | pullPolicy: IfNotPresent 23 | clusterConfig: 24 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 25 | vectorAggregatorConfigMapName: vector-aggregator-discovery 26 | {% endif %} 27 | credentialsSecret: test-airflow-credentials 28 | webservers: 29 | roleConfig: 30 | listenerClass: external-unstable 31 | config: 32 | logging: 33 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 34 | roleGroups: 35 | default: 36 | replicas: 1 37 | celeryExecutors: 38 | config: 39 | logging: 40 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 41 | roleGroups: 42 | default: 43 | replicas: 3 # ignored because paused 44 | schedulers: 45 | config: 46 | logging: 47 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 48 | roleGroups: 49 | default: 50 | replicas: 1 51 | -------------------------------------------------------------------------------- /docs/modules/airflow/pages/usage-guide/logging.adoc: -------------------------------------------------------------------------------- 1 | = Log aggregation 2 | :description: Forward Airflow logs to a Vector aggregator by configuring the ConfigMap and enabling the log agent. 3 | 4 | The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent: 5 | 6 | NOTE: The `task` handler is responsible for showing the task logs in the UI. 7 | Unfortunately, the log level of the `task` handler cannot be specified. 8 | To avoid that all logs are emitted to the UI, the log level of the `airflow.task` logger is set explicitly to `INFO`. 9 | You can change the log level as shown below. 10 | 11 | [source,yaml] 12 | ---- 13 | spec: 14 | clusterConfig: 15 | vectorAggregatorConfigMapName: vector-aggregator-discovery 16 | webservers: 17 | config: 18 | logging: 19 | enableVectorAgent: true 20 | containers: 21 | airflow: 22 | loggers: 23 | "flask_appbuilder": 24 | level: WARN 25 | celeryExecutors: 26 | config: 27 | logging: 28 | enableVectorAgent: true 29 | containers: 30 | airflow: 31 | loggers: 32 | "airflow.processor": 33 | level: INFO 34 | "airflow.task": 35 | level: DEBUG 36 | schedulers: 37 | config: 38 | logging: 39 | enableVectorAgent: true 40 | containers: 41 | airflow: 42 | loggers: 43 | "airflow.processor_manager": 44 | level: INFO 45 | ---- 46 | 47 | Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. 48 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | let 2 | self = import ./. {}; 3 | inherit (self) sources pkgs meta; 4 | 5 | beku = pkgs.callPackage (sources."beku.py" + "/beku.nix") {}; 6 | cargoDependencySetOfCrate = crate: [ crate ] ++ pkgs.lib.concatMap cargoDependencySetOfCrate (crate.dependencies ++ crate.buildDependencies); 7 | cargoDependencySet = pkgs.lib.unique (pkgs.lib.flatten (pkgs.lib.mapAttrsToList (crateName: crate: cargoDependencySetOfCrate crate.build) self.cargo.workspaceMembers)); 8 | in pkgs.mkShell rec { 9 | name = meta.operator.name; 10 | 11 | packages = with pkgs; [ 12 | ## cargo et-al 13 | rustup # this breaks pkg-config if it is in the nativeBuildInputs 14 | cargo-udeps 15 | 16 | ## Extra dependencies for use in a pure env (nix-shell --pure) 17 | ## These are mosuly useful for maintainers of this shell.nix 18 | ## to ensure all the dependencies are caught. 19 | # cacert 20 | # vim nvim nano 21 | ]; 22 | 23 | # derivation runtime dependencies 24 | buildInputs = pkgs.lib.concatMap (crate: crate.buildInputs) cargoDependencySet; 25 | 26 | # build time dependencies 27 | nativeBuildInputs = pkgs.lib.concatMap (crate: crate.nativeBuildInputs) cargoDependencySet ++ (with pkgs; [ 28 | beku 29 | docker 30 | gettext # for the proper envsubst 31 | git 32 | jq 33 | kind 34 | kubectl 35 | kubernetes-helm 36 | kuttl 37 | nix # this is implied, but needed in the pure env 38 | # tilt already defined in default.nix 39 | which 40 | yq-go 41 | ]); 42 | 43 | LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; 44 | BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include"; 45 | } 46 | --------------------------------------------------------------------------------