├── tests ├── templates │ ├── .gitkeep │ └── kuttl │ │ ├── orphaned_resources │ │ ├── 03-assert.yaml │ │ ├── 03-remove-rolegroup.yaml.j2 │ │ ├── 00-assert.yaml.j2 │ │ ├── 01-assert.yaml │ │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 02-assert.yaml │ │ ├── 03-errors.yaml │ │ ├── 01-install-zk.yaml.j2 │ │ └── 02-install-nifi.yaml.j2 │ │ ├── iceberg │ │ ├── 40-assert.yaml │ │ ├── 20-install-minio.yaml │ │ ├── 21-install-minio-jobs.yaml │ │ ├── 01-create-s3-connection.yaml │ │ ├── 21-assert.yaml │ │ ├── 61-assert.yaml │ │ ├── 70-assert.yaml │ │ ├── 32-install-hdfs.yaml.j2 │ │ ├── 33-install-hive.yaml.j2 │ │ ├── 34-assert.yaml │ │ ├── 34-install-trino.yaml.j2 │ │ ├── 50-install-nifi.yaml.j2 │ │ ├── 20-assert.yaml │ │ ├── 25-assert.yaml │ │ ├── 10-assert.yaml.j2 │ │ ├── 33-assert.yaml │ │ ├── 50-assert.yaml │ │ ├── 30-assert.yaml │ │ ├── 10-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 02-assert.yaml.j2 │ │ ├── 03-create-kerberos-secretclass.yaml.j2 │ │ ├── 25-install-hive-postgres.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 31-opa.yaml.j2 │ │ ├── 60-create-nifi-flow-configmap.yaml.j2 │ │ ├── 32-assert.yaml │ │ ├── 30-install-zookeeper.yaml.j2 │ │ ├── 00-rbac.yaml.j2 │ │ ├── 01_s3-connection.yaml │ │ ├── 25_helm-bitnami-postgresql-values.yaml.j2 │ │ ├── 40-create-iceberg-tables.j2 │ │ ├── 70-check-iceberg-tables.yaml.j2 │ │ ├── 03_kerberos-secretclass.yaml.j2 │ │ ├── 33_hive.yaml.j2 │ │ └── README.md │ │ ├── custom-components-git-sync │ │ ├── canvas.png │ │ ├── java-processors │ │ │ ├── nifi-sample-nar-1.0.0.nar │ │ │ └── sample-processor.tar.gz │ │ ├── 40-assert.yaml │ │ ├── 10-assert.yaml.j2 │ │ ├── 20-assert.yaml │ │ ├── 10-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 30-assert.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 40-test-nifi-greeting.yaml │ │ ├── python-processors │ │ │ └── greet_processor.py │ │ ├── 20-install-zk.yaml.j2 │ │ └── README.md │ │ ├── ldap │ │ ├── 20-test-nifi.yaml │ │ ├── 01-assert.yaml │ │ ├── 03-assert.yaml │ │ ├── 00-assert.yaml.j2 │ │ ├── 01-install-openldap.yaml │ │ ├── 10-assert.yaml │ │ ├── 12-assert.yaml │ │ ├── 20-assert.yaml │ │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 02-create-ldap-user.yaml │ │ ├── 11-create-authentication-classes.yaml.j2 │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 03-install-test-nifi.yaml │ │ ├── 02-assert.yaml │ │ ├── 10-install-zk.yaml.j2 │ │ ├── create-authentication-classes.yaml.j2 │ │ ├── create_ldap_user.sh │ │ └── 12-install-nifi.yaml.j2 │ │ ├── oidc-opa │ │ ├── 30-install-nifi.yaml │ │ ├── 40-create-configmap.yaml.j2 │ │ ├── 19-install-keycloak.yaml │ │ ├── 15-install-authentication-class.yaml │ │ ├── 20-assert.yaml │ │ ├── 45-assert.yaml │ │ ├── 19-assert.yaml │ │ ├── 00-assert.yaml.j2 │ │ ├── 10-assert.yaml │ │ ├── 30-assert.yaml │ │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 41-assert.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 15_authentication-class.yaml.j2 │ │ ├── 10-install-zk.yaml.j2 │ │ ├── 20-install-opa.yaml.j2 │ │ └── 41-install-test-container.yaml.j2 │ │ ├── external-access │ │ ├── 30-install-nifi.yaml │ │ ├── 15-create-listener-class.yaml │ │ ├── 15_listener-class.yaml │ │ ├── 00-range-limit.yaml │ │ ├── 10-assert.yaml.j2 │ │ ├── 20-assert.yaml │ │ ├── 10-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 00-rbac.yaml.j2 │ │ ├── 20-install-zk.yaml.j2 │ │ ├── 30-assert.yaml │ │ └── 30_nifi.yaml.j2 │ │ ├── logging │ │ ├── 06-test-log-aggregation.yaml │ │ ├── 06-assert.yaml │ │ ├── 00-assert.yaml.j2 │ │ ├── 05-assert.yaml │ │ ├── 01-assert.yaml │ │ ├── 02-assert.yaml │ │ ├── 03-create-configmap-with-prepared-logs.yaml │ │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 04-assert.yaml │ │ ├── 01-install-nifi-vector-aggregator.yaml │ │ ├── 05-install-nifi-test-runner.yaml │ │ ├── 02-install-zookeeper.yaml.j2 │ │ └── test_log_aggregation.py │ │ ├── smoke_v1 │ │ ├── 00-range-limit.yaml │ │ ├── 31-assert.yaml.j2 │ │ ├── 50-assert.yaml │ │ ├── 10-assert.yaml.j2 │ │ ├── 40-assert.yaml │ │ ├── 20-assert.yaml │ │ ├── 10-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 40-scale-up-nifi.yaml.j2 │ │ ├── 60-prepare-test-nifi.yaml │ │ ├── 33-assert.yaml │ │ ├── 60-assert.yaml.j2 │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 30-assert.yaml │ │ ├── 20-install-zookeeper.yaml.j2 │ │ ├── 50-install-test-nifi.yaml │ │ ├── 32-assert.yaml │ │ ├── cacert.pem │ │ └── test_nifi_metrics.py │ │ ├── smoke_v2 │ │ ├── 00-range-limit.yaml │ │ ├── 31-assert.yaml.j2 │ │ ├── 50-assert.yaml │ │ ├── 10-assert.yaml.j2 │ │ ├── 40-assert.yaml │ │ ├── 10-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 40-scale-up-nifi.yaml.j2 │ │ ├── 60-prepare-test-nifi.yaml │ │ ├── 33-assert.yaml │ │ ├── 20-assert.yaml.j2 │ │ ├── 60-assert.yaml.j2 │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 30-assert.yaml │ │ ├── 20-install-zookeeper.yaml.j2 │ │ ├── 32-assert.yaml │ │ ├── cacert.pem │ │ └── 50-install-test-nifi.yaml.j2 │ │ ├── upgrade │ │ ├── 03-assert.yaml │ │ ├── 00-assert.yaml.j2 │ │ ├── 01-assert.yaml │ │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 02-assert.yaml.j2 │ │ ├── 05-upgrade-nifi.yaml.j2 │ │ ├── 03-install-test-nifi.yaml │ │ ├── 05-assert.yaml.j2 │ │ ├── 04-prepare-test-nifi.yaml.j2 │ │ ├── 01-install-zk.yaml.j2 │ │ ├── 07-assert.yaml.j2 │ │ ├── 04-assert.yaml.j2 │ │ ├── cacert.pem │ │ ├── 02-install-nifi.yaml.j2 │ │ └── test_nifi_metrics.py │ │ ├── resources │ │ ├── 00-assert.yaml.j2 │ │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 01-assert.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 03-assert.yaml │ │ ├── 01-install-zk.yaml.j2 │ │ └── 02-assert.yaml.j2 │ │ └── cluster_operation │ │ ├── 00-assert.yaml.j2 │ │ ├── 10-assert.yaml │ │ ├── 00-install-vector-aggregator-discovery-configmap.yaml.j2 │ │ ├── 30-assert.yaml │ │ ├── 40-assert.yaml │ │ ├── 00-patch-ns.yaml.j2 │ │ ├── 20-assert.yaml │ │ ├── 50-assert.yaml │ │ ├── 10-install-zk.yaml.j2 │ │ ├── 30-stop-nifi.yaml.j2 │ │ ├── 40-pause-nifi.yaml.j2 │ │ ├── 50-restart-nifi.yaml.j2 │ │ └── 20-install-nifi.yaml.j2 ├── interu.yaml ├── release.yaml └── kuttl-test.yaml.jinja2 ├── docs ├── antora.yml ├── modules │ └── nifi │ │ ├── assets │ │ └── images │ │ │ ├── nifi-web-ui.png │ │ │ ├── listen-http-1.png │ │ │ ├── listen-http-2.png │ │ │ ├── listen-http-3.png │ │ │ ├── listen-http-4.png │ │ │ └── put-iceberg-processor.png │ │ ├── images │ │ └── listening-processor-example.png │ │ ├── examples │ │ └── getting_started │ │ │ ├── test_getting_started_stackablectl.sh │ │ │ ├── test_getting_started_helm.sh │ │ │ ├── install_output.txt │ │ │ └── install_output.txt.j2 │ │ ├── pages │ │ ├── reference │ │ │ ├── crds.adoc │ │ │ ├── index.adoc │ │ │ └── commandline-parameters.adoc │ │ ├── usage_guide │ │ │ ├── operations │ │ │ │ ├── pod-placement.adoc │ │ │ │ ├── cluster-operations.adoc │ │ │ │ ├── index.adoc │ │ │ │ ├── pod-disruptions.adoc │ │ │ │ └── graceful-shutdown.adoc │ │ │ ├── exposing-processors │ │ │ │ └── index.adoc │ │ │ ├── log-aggregation.adoc │ │ │ ├── listenerclass.adoc │ │ │ ├── extra-volumes.adoc │ │ │ ├── updating.adoc │ │ │ └── resource-configuration.adoc │ │ └── getting_started │ │ │ ├── index.adoc │ │ │ └── installation.adoc │ │ └── partials │ │ ├── supported-versions.adoc │ │ └── nav.adoc └── templating_vars.yaml ├── scripts ├── run_tests.sh ├── render_readme.sh ├── generate-manifests.sh └── docs_templating.sh ├── rust └── operator-binary │ ├── build.rs │ ├── src │ ├── operations │ │ ├── mod.rs │ │ ├── graceful_shutdown.rs │ │ └── pdb.rs │ ├── crd │ │ └── tls.rs │ └── security │ │ ├── mod.rs │ │ └── tls.rs │ └── Cargo.toml ├── .gitattributes ├── deploy ├── helm │ ├── chart_testing.yaml │ ├── ct.yaml │ └── nifi-operator │ │ ├── templates │ │ ├── configmap.yaml │ │ ├── service.yaml │ │ ├── _maintenance.tpl │ │ ├── serviceaccount.yaml │ │ └── _telemetry.tpl │ │ ├── Chart.yaml │ │ ├── .helmignore │ │ ├── README.md │ │ ├── configs │ │ └── properties.yaml │ │ └── values.yaml ├── stackable-operators-ns.yaml ├── DO_NOT_EDIT.md └── config-spec │ └── properties.yaml ├── .actionlint.yaml ├── .github ├── actionlint.yaml ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── normal-issue.md │ ├── 01-normal-issue.md │ ├── new_version.md │ └── 02-bug_report.yml ├── workflows │ ├── general_daily_security.yml │ └── pr_pre-commit.yaml ├── PULL_REQUEST_TEMPLATE │ ├── pre-release-getting-started-script.md │ └── pre-release-rust-deps.md └── pull_request_template.md ├── rust-toolchain.toml ├── .readme ├── static │ └── borrowed │ │ ├── stackable_overview.png │ │ └── Icon_Stackable.svg ├── partials │ ├── borrowed │ │ ├── header.md.j2 │ │ ├── related_reading.md.j2 │ │ ├── overview_blurb.md.j2 │ │ ├── documentation.md.j2 │ │ └── links.md.j2 │ └── main.md.j2 └── README.md.j2 ├── examples ├── entra-static-nifi-policies │ ├── canvas.png │ └── entra-redirect-uri.png └── simple-cluster │ └── simple-nifi-cluster.yaml ├── nix ├── meta.json ├── README.md └── sources.json ├── .envrc.sample ├── .vscode ├── settings.json └── launch.json ├── .pylintrc ├── .gitignore ├── renovate.json ├── .dockerignore ├── .hadolint.yaml ├── rustfmt.toml ├── .yamllint.yaml ├── .markdownlint.yaml ├── Cargo.toml ├── shell.nix ├── crate-hashes.json └── deny.toml /tests/templates/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/antora.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: home 3 | version: "nightly" 4 | -------------------------------------------------------------------------------- /scripts/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ./scripts/run-tests "$@" 4 | -------------------------------------------------------------------------------- /rust/operator-binary/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | built::write_built_file().unwrap(); 3 | } 4 | -------------------------------------------------------------------------------- /rust/operator-binary/src/operations/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod graceful_shutdown; 2 | pub mod pdb; 3 | pub mod upgrade; 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | nix/** linguist-generated 2 | Cargo.nix linguist-generated 3 | crate-hashes.json linguist-generated 4 | -------------------------------------------------------------------------------- /deploy/helm/chart_testing.yaml: -------------------------------------------------------------------------------- 1 | remote: origin 2 | target-branch: main 3 | chart-dirs: 4 | - deploy/helm 5 | all: true 6 | -------------------------------------------------------------------------------- /deploy/stackable-operators-ns.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: stackable-operators 6 | -------------------------------------------------------------------------------- /.actionlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | self-hosted-runner: 3 | # Ubicloud machines we are using 4 | labels: 5 | - ubicloud-standard-8-arm 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned_resources/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | -------------------------------------------------------------------------------- /.github/actionlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | self-hosted-runner: 3 | # Ubicloud machines we are using 4 | labels: 5 | - ubicloud-standard-8-arm 6 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | # DO NOT EDIT, this file is generated by operator-templating 2 | [toolchain] 3 | channel = "1.89.0" 4 | profile = "default" 5 | -------------------------------------------------------------------------------- /.readme/static/borrowed/stackable_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/.readme/static/borrowed/stackable_overview.png -------------------------------------------------------------------------------- /docs/modules/nifi/assets/images/nifi-web-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/docs/modules/nifi/assets/images/nifi-web-ui.png -------------------------------------------------------------------------------- /examples/entra-static-nifi-policies/canvas.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/examples/entra-static-nifi-policies/canvas.png -------------------------------------------------------------------------------- /nix/meta.json: -------------------------------------------------------------------------------- 1 | {"operator": {"name": "nifi-operator", "pretty_string": "Apache NiFi", "product_string": "nifi", "url": "stackabletech/nifi-operator.git"}} 2 | -------------------------------------------------------------------------------- /docs/modules/nifi/assets/images/listen-http-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/docs/modules/nifi/assets/images/listen-http-1.png -------------------------------------------------------------------------------- /docs/modules/nifi/assets/images/listen-http-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/docs/modules/nifi/assets/images/listen-http-2.png -------------------------------------------------------------------------------- /docs/modules/nifi/assets/images/listen-http-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/docs/modules/nifi/assets/images/listen-http-3.png -------------------------------------------------------------------------------- /docs/modules/nifi/assets/images/listen-http-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/docs/modules/nifi/assets/images/listen-http-4.png -------------------------------------------------------------------------------- /.envrc.sample: -------------------------------------------------------------------------------- 1 | # vim: syntax=conf 2 | # 3 | # If you use direnv, you can autoload the nix shell: 4 | # You will need to allow the directory the first time. 5 | use nix 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | name: create-iceberg-tables 6 | status: 7 | succeeded: 1 8 | -------------------------------------------------------------------------------- /docs/modules/nifi/assets/images/put-iceberg-processor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/docs/modules/nifi/assets/images/put-iceberg-processor.png -------------------------------------------------------------------------------- /docs/modules/nifi/images/listening-processor-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/docs/modules/nifi/images/listening-processor-example.png -------------------------------------------------------------------------------- /examples/entra-static-nifi-policies/entra-redirect-uri.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/examples/entra-static-nifi-policies/entra-redirect-uri.png -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/canvas.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/tests/templates/kuttl/custom-components-git-sync/canvas.png -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/20-install-minio.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: kubectl -n $NAMESPACE apply -f 20_minio.yaml 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/20-test-nifi.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: kubectl cp -n $NAMESPACE ./test_nifi.py test-nifi-0:/tmp 6 | -------------------------------------------------------------------------------- /docs/modules/nifi/examples/getting_started/test_getting_started_stackablectl.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "$0")" 5 | ./getting_started.sh stackablectl 6 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/reference/crds.adoc: -------------------------------------------------------------------------------- 1 | = CRD Reference 2 | 3 | Find all CRD reference for the Stackable Operator for Apache NiFi at: {crd-docs-base-url}/nifi-operator/{crd-docs-version}. 4 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/21-install-minio-jobs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: kubectl -n $NAMESPACE apply -f 21_minio_jobs.yaml 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/30-install-nifi.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: envsubst < 30_nifi.yaml | kubectl apply -n $NAMESPACE -f - 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/30-install-nifi.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: envsubst < 30_nifi.yaml | kubectl apply -n $NAMESPACE -f - 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/40-create-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: kuttl.dev/v1beta1 2 | kind: TestStep 3 | commands: 4 | - script: kubectl create cm test-script -n $NAMESPACE --from-file=test.py 5 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/19-install-keycloak.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: envsubst < 19_keycloak.yaml | kubectl apply -n $NAMESPACE -f - 6 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rust-analyzer.rustfmt.overrideCommand": [ 3 | "rustfmt", 4 | "+nightly-2025-10-23", 5 | "--edition", 6 | "2024", 7 | "--" 8 | ], 9 | } 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/15-create-listener-class.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: envsubst < 15_listener-class.yaml | kubectl apply -n $NAMESPACE -f - 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/01-create-s3-connection.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: envsubst '$NAMESPACE' < 01_s3-connection.yaml | kubectl apply -n $NAMESPACE -f - 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/06-test-log-aggregation.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: | 6 | kubectl cp ./test_log_aggregation.py $NAMESPACE/nifi-test-runner-0:/tmp 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/15-install-authentication-class.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: envsubst < 15_authentication-class.yaml | kubectl apply -n $NAMESPACE -f - 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl -n $NAMESPACE rollout status daemonset opa-server-default --timeout 300s 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/java-processors/nifi-sample-nar-1.0.0.nar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/tests/templates/kuttl/custom-components-git-sync/java-processors/nifi-sample-nar-1.0.0.nar -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/java-processors/sample-processor.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackabletech/nifi-operator/HEAD/tests/templates/kuttl/custom-components-git-sync/java-processors/sample-processor.tar.gz -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/15_listener-class.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: listeners.stackable.tech/v1alpha1 3 | kind: ListenerClass 4 | metadata: 5 | name: test-external-unstable-$NAMESPACE 6 | spec: 7 | serviceType: NodePort 8 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | 3 | # These rules are for missing docstrings which doesn't matter much for most of our simple scripts 4 | disable=C0114,C0115,C0116 5 | 6 | [FORMAT] 7 | 8 | max-line-length=999 9 | indent-string=' ' 10 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/header.md.j2: -------------------------------------------------------------------------------- 1 | 2 |

3 | Stackable Logo 4 |

5 | 6 |

{{title}}

7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tests/_work/ 2 | debug/ 3 | target/ 4 | **/*.rs.bk 5 | 6 | .idea/ 7 | *.iws 8 | *.iml 9 | 10 | *.tgz 11 | 12 | result 13 | image.tar 14 | 15 | tilt_options.json 16 | 17 | .direnv/ 18 | .direnvrc 19 | .envrc 20 | 21 | .DS_Store 22 | -------------------------------------------------------------------------------- /docs/templating_vars.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | helm: 3 | repo_name: sdp-charts 4 | repo_url: oci.stackable.tech 5 | versions: 6 | commons: 0.0.0-dev 7 | secret: 0.0.0-dev 8 | listener: 0.0.0-dev 9 | zookeeper: 0.0.0-dev 10 | nifi: 0.0.0-dev 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/21-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: batch/v1 7 | kind: Job 8 | metadata: 9 | name: minio-post-job 10 | status: 11 | succeeded: 1 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/45-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test 6 | timeout: 300 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE python-0 -- python /tmp/test-script/test.py 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned_resources/03-remove-rolegroup.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: nifi.stackable.tech/v1alpha1 3 | kind: NifiCluster 4 | metadata: 5 | name: test-nifi 6 | spec: 7 | nodes: 8 | roleGroups: 9 | throwaway: null 10 | -------------------------------------------------------------------------------- /docs/modules/nifi/examples/getting_started/test_getting_started_helm.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "$0")" 5 | ./getting_started.sh helm #ExternalIP commenting this out as the json parsing does not work with e.g. Kind 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/61-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: batch/v1 7 | kind: Job 8 | metadata: 9 | name: provision-nifi-flow 10 | status: 11 | succeeded: 1 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/70-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: batch/v1 7 | kind: Job 8 | metadata: 9 | name: check-iceberg-tables 10 | status: 11 | succeeded: 1 12 | -------------------------------------------------------------------------------- /docs/modules/nifi/examples/getting_started/install_output.txt: -------------------------------------------------------------------------------- 1 | Installed commons=0.0.0-dev operator 2 | Installed secret=0.0.0-dev operator 3 | Installed listener=0.0.0-dev operator 4 | Installed zookeeper=0.0.0-dev operator 5 | Installed nifi=0.0.0-dev operator 6 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | # We need to replace $NAMESPACE (by KUTTL) 6 | - script: envsubst '$NAMESPACE' < 32_hdfs.yaml | kubectl apply -n $NAMESPACE -f - 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/33-install-hive.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | # We need to replace $NAMESPACE (by KUTTL) 6 | - script: envsubst '$NAMESPACE' < 33_hive.yaml | kubectl apply -n $NAMESPACE -f - 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/34-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 720 5 | commands: 6 | - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/34-install-trino.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | # We need to replace $NAMESPACE (by KUTTL) 6 | - script: envsubst '$NAMESPACE' < 34_trino.yaml | kubectl apply -n $NAMESPACE -f - 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | # We need to replace $NAMESPACE (by KUTTL) 6 | - script: envsubst '$NAMESPACE' < 50_nifi.yaml | kubectl apply -n $NAMESPACE -f - 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/06-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: >- 6 | kubectl exec --namespace=$NAMESPACE nifi-test-runner-0 -- 7 | python /tmp/test_log_aggregation.py -n $NAMESPACE 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/00-range-limit.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: LimitRange 4 | metadata: 5 | name: limit-request-ratio 6 | spec: 7 | limits: 8 | - type: "Container" 9 | maxLimitRequestRatio: 10 | cpu: 5 11 | memory: 1 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/00-range-limit.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: LimitRange 4 | metadata: 5 | name: limit-request-ratio 6 | spec: 7 | limits: 8 | - type: "Container" 9 | maxLimitRequestRatio: 10 | cpu: 5 11 | memory: 1 12 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/related_reading.md.j2: -------------------------------------------------------------------------------- 1 | 2 | {%- if related_reading_links -%} 3 | ## Related Reading 4 | {% for (text, link) in related_reading_links %} 5 | * [{{text}}]({{link}}) 6 | {%- endfor %} 7 | {%- endif -%} 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: batch/v1 7 | kind: Job 8 | metadata: 9 | name: test-nifi-greeting 10 | status: 11 | succeeded: 1 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: Deployment 8 | metadata: 9 | name: minio 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: openldap 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/00-range-limit.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: LimitRange 4 | metadata: 5 | name: limit-request-ratio 6 | spec: 7 | limits: 8 | - type: "Container" 9 | maxLimitRequestRatio: 10 | cpu: 5 11 | memory: 1 12 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/25-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: postgresql 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/19-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: Deployment 8 | metadata: 9 | name: keycloak 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/31-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 30 5 | commands: 6 | - script: kubectl get cm -n $NAMESPACE nifi-node-default -o yaml | grep -- 'nifi.web.proxy.host=.*example.com:1234' | xargs test ! -z 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/31-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 30 5 | commands: 6 | - script: kubectl get cm -n $NAMESPACE nifi-node-default -o yaml | grep -- 'nifi.web.proxy.host=.*example.com:1234' | xargs test ! -z 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/01-install-openldap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | # We need to replace $NAMESPACE (by KUTTL) in the install-openldap.yaml 6 | - script: eval "echo \"$(cat install-openldap.yaml)\"" | kubectl apply -f - 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/05-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: nifi-test-runner 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/33-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 900 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: hive-metastore-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: nifi-node-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/12-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi-node-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: nifi-vector-aggregator 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: nifi-node-default 10 | status: 11 | readyReplicas: 3 12 | replicas: 3 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: nifi-node-default 10 | status: 11 | readyReplicas: 3 12 | replicas: 3 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: zookeeper-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/03-create-configmap-with-prepared-logs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: > 6 | kubectl create configmap prepared-logs 7 | --from-file=prepared-logs.log4j.xml 8 | --namespace=$NAMESPACE 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi-node-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned_resources/00-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: zookeeper-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/10-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi.py -u integrationtest -p 'bindPasswordWithSpecialCharacter\@<&>"'"'" -n $NAMESPACE -c 2 7 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/10-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 5 | --- 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: vector-aggregator-discovery 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned_resources/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /deploy/helm/ct.yaml: -------------------------------------------------------------------------------- 1 | # This file is used for chart-testing (https://github.com/helm/chart-testing) 2 | # The name "ct.yaml" is not very self-descriptive but it is the default that chart-testing is looking for 3 | --- 4 | remote: origin 5 | target-branch: main 6 | chart-dirs: 7 | - deploy/helm 8 | all: true 9 | -------------------------------------------------------------------------------- /deploy/helm/nifi-operator/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | data: 4 | {{ (.Files.Glob "configs/*").AsConfig | indent 2 }} 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ include "operator.fullname" . }}-configmap 8 | labels: 9 | {{- include "operator.labels" . | nindent 4 }} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/operations/pod-placement.adoc: -------------------------------------------------------------------------------- 1 | = Pod placement 2 | 3 | You can configure the Pod placement of the NiFi pods as described in xref:concepts:operations/pod_placement.adoc[]. 4 | 5 | The default affinities created by the operator are: 6 | 7 | 1. Distribute all the NiFi Pods (weight 70) 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/10-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/10-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /docs/modules/nifi/examples/getting_started/install_output.txt.j2: -------------------------------------------------------------------------------- 1 | Installed commons={{ versions.commons }} operator 2 | Installed secret={{ versions.secret }} operator 3 | Installed listener={{ versions.listener }} operator 4 | Installed zookeeper={{ versions.zookeeper }} operator 5 | Installed nifi={{ versions.nifi }} operator 6 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/exposing-processors/index.adoc: -------------------------------------------------------------------------------- 1 | = Exposing processors 2 | 3 | There are two mechanisms to expose Nifi processors to the outside world. 4 | 5 | 1. xref:nifi:usage_guide/exposing-processors/http.adoc[HTTP traffic] 6 | 2. xref:nifi:usage_guide/exposing-processors/tcp.adoc[Generic TCP traffic] 7 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/operations/cluster-operations.adoc: -------------------------------------------------------------------------------- 1 | = Cluster operation 2 | 3 | Apache NiFi installations can be configured with different cluster operations such as pausing reconciliation or stopping the cluster. 4 | See xref:concepts:operations/cluster_operations.adoc[cluster operations] for more details. 5 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/02-create-ldap-user.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: create-ldap-user 6 | commands: 7 | - script: kubectl cp -n $NAMESPACE ./create_ldap_user.sh openldap-0:/tmp 8 | - script: kubectl exec -n $NAMESPACE openldap-0 -- sh /tmp/create_ldap_user.sh 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/41-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-test-container 6 | timeout: 300 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: python 12 | status: 13 | readyReplicas: 1 14 | replicas: 1 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/10-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned_resources/00-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/40-scale-up-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | kubectl --namespace $NAMESPACE 7 | patch nificlusters.nifi.stackable.tech nifi 8 | --type=merge --patch '{"spec":{"nodes": {"roleGroups": {"default": {"replicas": 3}}}}}' 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/40-scale-up-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | kubectl --namespace $NAMESPACE 7 | patch nificlusters.nifi.stackable.tech nifi 8 | --type=merge --patch '{"spec":{"nodes": {"roleGroups": {"default": {"replicas": 3}}}}}' 9 | -------------------------------------------------------------------------------- /deploy/helm/nifi-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v2 3 | name: nifi-operator 4 | version: "0.0.0-dev" 5 | appVersion: "0.0.0-dev" 6 | description: The Stackable Operator for Apache NiFi 7 | home: https://github.com/stackabletech/nifi-operator 8 | maintainers: 9 | - name: Stackable 10 | url: https://www.stackable.tech 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/10-install-vector-aggregator-discovery-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: vector-aggregator-discovery 7 | data: 8 | ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "local>stackabletech/.github:renovate-config" 5 | ], 6 | "ignorePaths": [".github/workflows/build.yaml", ".github/workflows/general_daily_security.yml", ".github/workflows/integration-test.yml", ".github/workflows/pr_pre-commit.yaml"] 7 | } 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/60-prepare-test-nifi.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: kubectl cp -n $NAMESPACE ./test_nifi_metrics.py test-nifi-0:/tmp 6 | - script: kubectl cp -n $NAMESPACE ./test_nifi.py test-nifi-0:/tmp 7 | - script: kubectl cp -n $NAMESPACE ./cacert.pem test-nifi-0:/tmp 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/60-prepare-test-nifi.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: kubectl cp -n $NAMESPACE ./test_nifi_metrics.py test-nifi-0:/tmp 6 | - script: kubectl cp -n $NAMESPACE ./test_nifi.py test-nifi-0:/tmp 7 | - script: kubectl cp -n $NAMESPACE ./cacert.pem test-nifi-0:/tmp 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/02-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | {% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} 6 | --- 7 | apiVersion: apps/v1 8 | kind: StatefulSet 9 | metadata: 10 | name: krb5-kdc 11 | status: 12 | readyReplicas: 1 13 | replicas: 1 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/11-create-authentication-classes.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | metadata: 5 | name: create-ldap-user 6 | commands: 7 | # We need to replace $NAMESPACE (by KUTTL) in the create-authentication-classes.yaml(.j2) 8 | - script: eval "echo \"$(cat create-authentication-classes.yaml)\"" | kubectl apply -f - 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | {% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestStep 5 | commands: 6 | # We need to replace $NAMESPACE (by KUTTL) 7 | - script: envsubst '$NAMESPACE' < 03_kerberos-secretclass.yaml | kubectl apply -n $NAMESPACE -f - 8 | {% endif %} 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/33-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # This test checks if the containerdebug-state.json file is present and valid 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | timeout: 600 6 | commands: 7 | - script: kubectl exec -n $NAMESPACE --container nifi nifi-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/33-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # This test checks if the containerdebug-state.json file is present and valid 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | timeout: 600 6 | commands: 7 | - script: kubectl exec -n $NAMESPACE --container nifi nifi-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' 8 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/operations/index.adoc: -------------------------------------------------------------------------------- 1 | = Operations 2 | 3 | This section of the documentation is intended for the operations teams that maintain a Stackable Data Platform installation. 4 | 5 | Please read the xref:concepts:operations/index.adoc[Concepts page on Operations] that contains the necessary details to operate the platform in a production environment. 6 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/overview_blurb.md.j2: -------------------------------------------------------------------------------- 1 | 2 | It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka, Apache Druid, Trino or Apache Spark, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms). 3 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/01-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-zk-server-default 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: test-nifi-znode 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/20-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['use-zookeeper-manager'] == 'true' %} 2 | --- 3 | apiVersion: kuttl.dev/v1beta1 4 | kind: TestAssert 5 | timeout: 600 6 | --- 7 | apiVersion: apps/v1 8 | kind: StatefulSet 9 | metadata: 10 | name: zookeeper-server-default 11 | status: 12 | readyReplicas: 1 13 | replicas: 1 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi-node-default 10 | spec: 11 | template: 12 | spec: 13 | terminationGracePeriodSeconds: 300 14 | status: 15 | readyReplicas: 2 16 | replicas: 2 17 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/60-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi.py -u admin -p 'passwordWithSpecialCharacter\@<&>"'"'" -n $NAMESPACE -c 3 7 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE 8 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/60-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi.py -u admin -p 'passwordWithSpecialCharacter\@<&>"'"'" -n $NAMESPACE -c 3 7 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE 8 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/reference/index.adoc: -------------------------------------------------------------------------------- 1 | = Reference 2 | 3 | Consult the reference documentation section to find exhaustive information on: 4 | 5 | * Descriptions and default values of all properties in the CRDs used by this operator in the xref:reference/crds.adoc[]. 6 | * The xref:reference/commandline-parameters.adoc[] and xref:reference/environment-variables.adoc[] accepted by the operator. 7 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | debug/ 2 | target/ 3 | **/*.rs.bk 4 | 5 | .idea/ 6 | *.iws 7 | 8 | Cargo.nix 9 | crate-hashes.json 10 | result 11 | image.tar 12 | 13 | # We do NOT want to ignore .git because we use the `built` crate to gather the current git commit hash at built time 14 | # This means we need the .git directory in our Docker image, it will be thrown away and won't be included in the final image 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/25-install-hive-postgres.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | timeout: 300 5 | commands: 6 | - script: >- 7 | helm upgrade postgresql 8 | --install 9 | --version=12.5.6 10 | --namespace $NAMESPACE 11 | -f 25_helm-bitnami-postgresql-values.yaml 12 | --repo https://charts.bitnami.com/bitnami postgresql 13 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl -n $NAMESPACE wait --for=condition=stopped nificlusters.nifi.stackable.tech/test-nifi --timeout 301s 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-nifi-node-default 12 | status: 13 | replicas: 0 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/40-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl -n $NAMESPACE wait --for=condition=reconciliationPaused nificlusters.nifi.stackable.tech/test-nifi --timeout 301s 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-nifi-node-default 12 | status: 13 | replicas: 0 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned_resources/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/00-patch-ns.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['openshift'] == 'true' %} 2 | # see https://github.com/stackabletech/issues/issues/566 3 | --- 4 | apiVersion: kuttl.dev/v1beta1 5 | kind: TestStep 6 | commands: 7 | - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' 8 | timeout: 120 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/20-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | commands: 6 | - script: kubectl -n $NAMESPACE wait --for=condition=available nificlusters.nifi.stackable.tech/test-nifi --timeout 1201s 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-nifi-node-default 12 | status: 13 | readyReplicas: 2 14 | replicas: 2 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/50-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | commands: 6 | - script: kubectl -n $NAMESPACE wait --for=condition=available nificlusters.nifi.stackable.tech/test-nifi --timeout 601s 7 | --- 8 | apiVersion: apps/v1 9 | kind: StatefulSet 10 | metadata: 11 | name: test-nifi-node-default 12 | status: 13 | readyReplicas: 2 14 | replicas: 2 15 | -------------------------------------------------------------------------------- /scripts/render_readme.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Check if jinja2 is there 5 | if ! command -v jinja2 &> /dev/null 6 | then 7 | echo "jinja2 could not be found. Use 'pip install jinja2-cli' to install it." 8 | exit 1 9 | fi 10 | 11 | SCRIPT_DIR=$(dirname "$0") 12 | cd "$SCRIPT_DIR/../.readme" 13 | jinja2 README.md.j2 -o ../README.md 14 | cd .. 15 | 16 | python3 scripts/ensure_one_trailing_newline.py README.md 17 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned_resources/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi-node-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: test-nifi-node-throwaway 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/04-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi-node-automatic-log-config 10 | status: 11 | readyReplicas: 1 12 | replicas: 1 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: test-nifi-node-custom-log-config 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | blank_issues_enabled: true 3 | contact_links: 4 | - name: 🙋🏾 Question 5 | about: Use this to ask a question about this project 6 | url: https://github.com/orgs/stackabletech/discussions/new?category=q-a 7 | - name: 🚀 Feature Requests and other things 8 | about: Open an issue with your feature request or any other issue not covered elsewhere 9 | url: https://github.com/stackabletech/nifi-operator/issues/new 10 | -------------------------------------------------------------------------------- /.hadolint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ignored: 3 | # Warning: Use the -y switch to avoid manual input dnf install -y 4 | # https://github.com/hadolint/hadolint/wiki/DL3038 5 | # Reason: We set `assumeyes=True` in dnf.conf in our base image 6 | - DL3038 7 | 8 | # Warning: Specify version with dnf install -y - 9 | # https://github.com/hadolint/hadolint/wiki/DL3041 10 | # Reason: It's good advice, but we're not set up to pin versions just yet 11 | - DL3041 12 | -------------------------------------------------------------------------------- /deploy/DO_NOT_EDIT.md: -------------------------------------------------------------------------------- 1 | # DO NOT EDIT 2 | 3 | These Helm charts and manifests are automatically generated. 4 | Please do not edit anything except for files explicitly mentioned below in this 5 | directory manually. 6 | 7 | The following files are ok to edit: 8 | 9 | - helm/nifi-operator/templates/roles.yaml 10 | - helm/nifi-operator/values.yaml 11 | 12 | The details are in-motion but check this repository for a few details: 13 | 14 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/02-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi-node-default 10 | status: 11 | readyReplicas: 3 12 | replicas: 3 13 | --- 14 | apiVersion: nifi.stackable.tech/v1alpha1 15 | kind: NifiCluster 16 | metadata: 17 | name: test-nifi 18 | status: 19 | deployed_version: {{ test_scenario['values']['nifi_old'].split(',')[0] }} 20 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # This file includes unstable features, so you need to run "cargo +nightly fmt" to format your code. 2 | # It's also ok to use the stable toolchain by simple running "cargo fmt", but using the nigthly formatter is prefered. 3 | 4 | # https://doc.rust-lang.org/nightly/edition-guide/rust-2024/rustfmt-style-edition.html 5 | style_edition = "2024" 6 | imports_granularity = "Crate" 7 | group_imports = "StdExternalCrate" 8 | reorder_impl_items = true 9 | use_field_init_shorthand = true 10 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/operations/pod-disruptions.adoc: -------------------------------------------------------------------------------- 1 | = Allowed Pod disruptions 2 | 3 | You can configure the permitted Pod disruptions for NiFi nodes as described in xref:concepts:operations/pod_disruptions.adoc[]. 4 | 5 | Unless you configure something else or disable the provided PodDisruptionBudgets (PDBs), the following PDBs are written: 6 | 7 | == Nodes 8 | The provided PDBs only allow a single node to be offline at any given time, regardless of the number of replicas or `roleGroups`. 9 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/05-upgrade-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: nifi.stackable.tech/v1alpha1 3 | kind: NifiCluster 4 | metadata: 5 | name: test-nifi 6 | spec: 7 | image: 8 | {% if test_scenario['values']['nifi_new'].find(",") > 0 %} 9 | custom: "{{ test_scenario['values']['nifi_new'].split(',')[1] }}" 10 | productVersion: "{{ test_scenario['values']['nifi_new'].split(',')[0] }}" 11 | {% else %} 12 | custom: null 13 | productVersion: "{{ test_scenario['values']['nifi_new'] }}" 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/01-install-nifi-vector-aggregator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: >- 6 | helm install nifi-vector-aggregator vector 7 | --namespace $NAMESPACE 8 | --version 0.45.0 9 | --repo https://helm.vector.dev 10 | --values nifi-vector-aggregator-values.yaml 11 | --- 12 | apiVersion: v1 13 | kind: ConfigMap 14 | metadata: 15 | name: nifi-vector-aggregator-discovery 16 | data: 17 | ADDRESS: nifi-vector-aggregator:6123 18 | -------------------------------------------------------------------------------- /.yamllint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | ignore: | 5 | deploy/helm/**/templates 6 | 7 | rules: 8 | line-length: disable 9 | truthy: 10 | check-keys: false 11 | comments: 12 | min-spaces-from-content: 1 # Needed due to https://github.com/adrienverge/yamllint/issues/443 13 | indentation: 14 | indent-sequences: consistent 15 | comments-indentation: disable # This is generally useless and interferes with commented example values 16 | braces: 17 | max-spaces-inside: 1 18 | max-spaces-inside-empty: 0 19 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/03-install-test-nifi.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-nifi 6 | labels: 7 | app: test-nifi 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-nifi 13 | template: 14 | metadata: 15 | labels: 16 | app: test-nifi 17 | spec: 18 | containers: 19 | - name: test-nifi 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | command: ["sleep", "infinity"] 22 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/31-opa.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: opa.stackable.tech/v1alpha1 3 | kind: OpaCluster 4 | metadata: 5 | name: opa 6 | spec: 7 | image: 8 | {% if test_scenario['values']['opa-l'].find(",") > 0 %} 9 | custom: "{{ test_scenario['values']['opa-l'].split(',')[1] }}" 10 | productVersion: "{{ test_scenario['values']['opa-l'].split(',')[0] }}" 11 | {% else %} 12 | productVersion: "{{ test_scenario['values']['opa-l'] }}" 13 | {% endif %} 14 | servers: 15 | roleGroups: 16 | default: 17 | replicas: 1 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/03-install-test-nifi.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-nifi 6 | labels: 7 | app: test-nifi 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-nifi 13 | template: 14 | metadata: 15 | labels: 16 | app: test-nifi 17 | spec: 18 | containers: 19 | - name: test-nifi 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | stdin: true 22 | tty: true 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned_resources/03-errors.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-nifi-node-throwaway 6 | --- 7 | apiVersion: v1 8 | kind: Pod 9 | metadata: 10 | name: test-nifi-node-throwaway-0 11 | --- 12 | apiVersion: v1 13 | kind: ConfigMap 14 | metadata: 15 | name: test-nifi-node-throwaway 16 | --- 17 | apiVersion: v1 18 | kind: ConfigMap 19 | metadata: 20 | name: test-nifi-node-throwaway-log 21 | --- 22 | apiVersion: v1 23 | kind: Service 24 | metadata: 25 | name: test-nifi-node-throwaway 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: nifi-node-default 10 | spec: 11 | template: 12 | spec: 13 | terminationGracePeriodSeconds: 300 14 | status: 15 | readyReplicas: 2 16 | replicas: 2 17 | --- 18 | apiVersion: policy/v1 19 | kind: PodDisruptionBudget 20 | metadata: 21 | name: nifi-node 22 | status: 23 | expectedPods: 2 24 | currentHealthy: 2 25 | disruptionsAllowed: 1 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: nifi-node-default 10 | spec: 11 | template: 12 | spec: 13 | terminationGracePeriodSeconds: 300 14 | status: 15 | readyReplicas: 2 16 | replicas: 2 17 | --- 18 | apiVersion: policy/v1 19 | kind: PodDisruptionBudget 20 | metadata: 21 | name: nifi-node 22 | status: 23 | expectedPods: 2 24 | currentHealthy: 2 25 | disruptionsAllowed: 1 26 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "lldb", 6 | "request": "launch", 7 | "name": "Debug operator binary", 8 | "cargo": { 9 | "args": ["build"], 10 | "filter": { 11 | "name": "stackable-{[ operator.name }]", 12 | "kind": "bin" 13 | } 14 | }, 15 | "args": ["run"], 16 | "cwd": "${workspaceFolder}" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/40-test-nifi-greeting.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | name: test-nifi-greeting 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: test 11 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 12 | command: 13 | - /bin/bash 14 | - -c 15 | args: 16 | - test "$(curl nifi-greeting/greeting)" = HELLO! 17 | restartPolicy: OnFailure 18 | terminationGracePeriodSeconds: 1 19 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | {% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} 6 | - script: cat 60_nifi-flow-with-kerberos.json | envsubst '$NAMESPACE' | kubectl -n $NAMESPACE create configmap nifi-flow --from-file=nifi-flow.json=/dev/stdin 7 | {% else %} 8 | - script: cat 60_nifi-flow-without-kerberos.json | envsubst '$NAMESPACE' | kubectl -n $NAMESPACE create configmap nifi-flow --from-file=nifi-flow.json=/dev/stdin 9 | 10 | {% endif %} 11 | -------------------------------------------------------------------------------- /tests/interu.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | runners: 3 | amd64: 4 | platform: aks-1.32 5 | ttl: 6h 6 | node-groups: 7 | - name: default 8 | arch: amd64 9 | size: medium 10 | disk-gb: 100 11 | nodes: 3 12 | 13 | profiles: 14 | # TODO (@Techassi): This will be enabled later 15 | # schedule: 16 | # strategy: use-runner 17 | # runner: amd64 18 | # options: 19 | # beku-parallelism: 2 20 | smoke-latest: 21 | strategy: use-runner 22 | runner: amd64 23 | options: 24 | beku-parallelism: 2 25 | beku-test-suite: smoke-latest 26 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/05-install-nifi-test-runner.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: nifi-test-runner 6 | labels: 7 | app: nifi-test-runner 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: nifi-test-runner 13 | template: 14 | metadata: 15 | labels: 16 | app: nifi-test-runner 17 | spec: 18 | containers: 19 | - name: nifi-test-runner 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | stdin: true 22 | tty: true 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/32-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 900 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: hdfs-namenode-default 10 | status: 11 | readyReplicas: 2 12 | replicas: 2 13 | --- 14 | apiVersion: apps/v1 15 | kind: StatefulSet 16 | metadata: 17 | name: hdfs-journalnode-default 18 | status: 19 | readyReplicas: 1 20 | replicas: 1 21 | --- 22 | apiVersion: apps/v1 23 | kind: StatefulSet 24 | metadata: 25 | name: hdfs-datanode-default 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/03-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: check-jvm-heap-args 6 | timeout: 600 7 | commands: 8 | - script: kubectl get cm -n $NAMESPACE test-nifi-node-resources-from-role -o yaml | grep -E 'java.arg..=-Xmx1638m' | xargs test ! -z 9 | --- 10 | apiVersion: kuttl.dev/v1beta1 11 | kind: TestAssert 12 | metadata: 13 | name: check-jvm-heap-args 14 | timeout: 600 15 | commands: 16 | - script: kubectl get cm -n $NAMESPACE test-nifi-node-resources-from-role-group -o yaml | grep -E 'java.arg..=-Xms2457m' | xargs test ! -z 17 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/02-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | commands: 5 | - script: kubectl exec -n $NAMESPACE openldap-0 -- ldapsearch -H ldap://localhost:1389 -D "cn=integrationtest,ou=my users,dc=example,dc=org" -w 'bindPasswordWithSpecialCharacter\@<&>"'"'" -b "ou=my users,dc=example,dc=org" > /dev/null 6 | - script: kubectl exec -n $NAMESPACE openldap-0 -- bash -c LDAPTLS_CACERT=/tls/ca.crt ldapsearch -Z -H ldaps://localhost:1636 -D "cn=integrationtest,ou=my users,dc=example,dc=org" -w 'bindPasswordWithSpecialCharacter\@<&>"'"'" -b "ou=my users,dc=example,dc=org" > /dev/null 7 | -------------------------------------------------------------------------------- /docs/modules/nifi/partials/supported-versions.adoc: -------------------------------------------------------------------------------- 1 | // The version ranges supported by NiFi-Operator 2 | // This is a separate file, since it is used by both the direct NiFi-Operator documentation, and the overarching 3 | // Stackable Platform documentation. 4 | 5 | * 2.6.0 (LTS, Please note that you need to upgrade to at least 1.27.x before upgrading to 2.x.x!) 6 | * 2.4.0 (Deprecated, Please note that you need to upgrade to at least 1.27.x before upgrading to 2.x.x!) 7 | * 1.28.1 (Deprecated) 8 | * 1.27.0 (Deprecated) 9 | 10 | For details on how to upgrade your NiFi version, refer to xref:nifi:usage_guide/updating.adoc[]. 11 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/python-processors/greet_processor.py: -------------------------------------------------------------------------------- 1 | from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult 2 | 3 | 4 | class Greet(FlowFileTransform): 5 | class Java: 6 | implements = ["org.apache.nifi.python.processor.FlowFileTransform"] 7 | 8 | class ProcessorDetails: 9 | version = "1.0.0" 10 | description = "A Python processor that greets politely." 11 | 12 | def __init__(self, **kwargs): 13 | pass 14 | 15 | def transform(self, context, flowfile): 16 | return FlowFileTransformResult(relationship="success", contents="Hello!") 17 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/05-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 1200 5 | --- 6 | apiVersion: apps/v1 7 | kind: StatefulSet 8 | metadata: 9 | name: test-nifi-node-default 10 | status: 11 | readyReplicas: 3 12 | replicas: 3 13 | --- 14 | apiVersion: nifi.stackable.tech/v1alpha1 15 | kind: NifiCluster 16 | metadata: 17 | name: test-nifi 18 | status: 19 | {% if test_scenario['values']['nifi_new'].find(",") > 0 %} 20 | deployed_version: "{{ test_scenario['values']['nifi_new'].split(',')[0] }}" 21 | {% else %} 22 | deployed_version: {{ test_scenario['values']['nifi_new'] }} 23 | {% endif %} 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: zookeeper 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | -------------------------------------------------------------------------------- /deploy/helm/nifi-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # ============= 2 | # This file is automatically generated from the templates in stackabletech/operator-templating 3 | # DON'T MANUALLY EDIT THIS FILE 4 | # ============= 5 | 6 | # Patterns to ignore when building packages. 7 | # This supports shell glob matching, relative path matching, and 8 | # negation (prefixed with !). Only one pattern per line. 9 | .DS_Store 10 | # Common VCS dirs 11 | .git/ 12 | .gitignore 13 | .bzr/ 14 | .bzrignore 15 | .hg/ 16 | .hgignore 17 | .svn/ 18 | # Common backup files 19 | *.swp 20 | *.bak 21 | *.tmp 22 | *.orig 23 | *~ 24 | # Various IDEs 25 | .project 26 | .idea/ 27 | *.tmproj 28 | .vscode/ 29 | -------------------------------------------------------------------------------- /deploy/helm/nifi-operator/templates/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | # Note(@sbernauer): We could also call the Service something like 7 | # "product-operator-conversion-webhook". However, in the future we will have more webhooks, and 8 | # it seems like an overkill to have a dedicated Service per webhook. 9 | name: {{ include "operator.fullname" . }} 10 | labels: 11 | {{- include "operator.labels" . | nindent 4 }} 12 | spec: 13 | selector: 14 | {{- include "operator.selectorLabels" . | nindent 6 }} 15 | ports: 16 | - name: conversion-webhook 17 | protocol: TCP 18 | port: 8443 19 | targetPort: 8443 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/normal-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Normal issue 3 | about: This is just a normal empty issue with a simple checklist 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Issue checklist 11 | 12 | This is a simple checklist of things to bear in mind when creating a new issue. 13 | 14 | - [ ] Describe the use-case, as far is possible. For instance, using the pattern "As a XXXX, I would like XXXX to be able to do XXXX" helps to identify the feature as well as the problem it is intended to address. 15 | - [ ] Indicate an approximate level of importance and urgency. 16 | - [ ] Indicate if there is a known work-around until such time as the issue has been implemented. 17 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/log-aggregation.adoc: -------------------------------------------------------------------------------- 1 | = Log aggregation 2 | :description: The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent. 3 | 4 | The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent: 5 | 6 | [source,yaml] 7 | ---- 8 | spec: 9 | clusterConfig: 10 | vectorAggregatorConfigMapName: vector-aggregator-discovery 11 | nodes: 12 | config: 13 | logging: 14 | enableVectorAgent: true 15 | ---- 16 | 17 | Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. 18 | -------------------------------------------------------------------------------- /.readme/README.md.j2: -------------------------------------------------------------------------------- 1 | {%- set title="Stackable Operator for Apache NiFi" -%} 2 | {%- set operator_name="nifi" -%} 3 | {%- set operator_docs_slug="nifi" -%} 4 | {%- set related_reading_links=[] -%} 5 | 6 | {% filter trim %} 7 | {%- include "partials/borrowed/header.md.j2" -%} 8 | {% endfilter %} 9 | 10 | {% filter trim %} 11 | {%- include "partials/borrowed/links.md.j2" -%} 12 | {% endfilter %} 13 | 14 | {% filter trim %} 15 | {%- include "partials/main.md.j2" -%} 16 | {% endfilter %} 17 | 18 | {% filter trim %} 19 | {%- include "partials/borrowed/footer.md.j2" -%} 20 | {% endfilter %} 21 | 22 | {% filter trim %} 23 | {%- include "partials/borrowed/related_reading.md.j2" -%} 24 | {% endfilter %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/04-prepare-test-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: kubectl cp -n $NAMESPACE ./test_nifi_metrics.py test-nifi-0:/tmp 6 | - script: kubectl cp -n $NAMESPACE ./test_nifi.py test-nifi-0:/tmp 7 | - script: kubectl cp -n $NAMESPACE ./flow.py test-nifi-0:/tmp 8 | {% if test_scenario['values']['nifi_old'].split(',')[0] == '2.0.0' %} 9 | - script: kubectl cp -n $NAMESPACE ./generate-and-log-flowfiles.json test-nifi-0:/tmp 10 | {% else %} 11 | - script: kubectl cp -n $NAMESPACE ./generate-and-log-flowfiles.xml test-nifi-0:/tmp 12 | {% endif %} 13 | - script: kubectl cp -n $NAMESPACE ./cacert.pem test-nifi-0:/tmp 14 | -------------------------------------------------------------------------------- /.github/workflows/general_daily_security.yml: -------------------------------------------------------------------------------- 1 | # ============= 2 | # This file is automatically generated from the templates in stackabletech/operator-templating 3 | # DON'T MANUALLY EDIT THIS FILE 4 | # ============= 5 | --- 6 | name: Daily Security Audit 7 | 8 | on: 9 | schedule: 10 | - cron: '15 4 * * *' 11 | workflow_dispatch: 12 | 13 | permissions: {} 14 | 15 | jobs: 16 | audit: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 20 | with: 21 | persist-credentials: false 22 | - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 23 | with: 24 | token: ${{ secrets.GITHUB_TOKEN }} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/20-install-zookeeper.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: zookeeper 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper'] }}" 9 | pullPolicy: IfNotPresent 10 | clusterConfig: 11 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleConfig: 19 | listenerClass: {{ test_scenario['values']['listener-class'] }} 20 | roleGroups: 21 | default: 22 | replicas: 1 23 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/00-rbac.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | kind: Role 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: test-role 6 | rules: 7 | {% if test_scenario['values']['openshift'] == "true" %} 8 | - apiGroups: ["security.openshift.io"] 9 | resources: ["securitycontextconstraints"] 10 | resourceNames: ["privileged"] 11 | verbs: ["use"] 12 | {% endif %} 13 | --- 14 | apiVersion: v1 15 | kind: ServiceAccount 16 | metadata: 17 | name: test-sa 18 | --- 19 | kind: RoleBinding 20 | apiVersion: rbac.authorization.k8s.io/v1 21 | metadata: 22 | name: test-rb 23 | subjects: 24 | - kind: ServiceAccount 25 | name: test-sa 26 | roleRef: 27 | kind: Role 28 | name: test-role 29 | apiGroup: rbac.authorization.k8s.io 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/50-install-test-nifi.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-nifi 6 | labels: 7 | app: test-nifi 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-nifi 13 | template: 14 | metadata: 15 | labels: 16 | app: test-nifi 17 | spec: 18 | containers: 19 | - name: test-nifi 20 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 21 | command: ["sleep", "infinity"] 22 | resources: 23 | requests: 24 | memory: "128Mi" 25 | cpu: "100m" 26 | limits: 27 | memory: "128Mi" 28 | cpu: "400m" 29 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/listenerclass.adoc: -------------------------------------------------------------------------------- 1 | = Service exposition with ListenerClasses 2 | :description: Configure Apache NiFi service exposure with cluster-internal or external-unstable listener classes. 3 | 4 | The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Node pod. 5 | The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.roleConfig.listenerClass`: 6 | 7 | [source,yaml] 8 | ---- 9 | spec: 10 | nodes: 11 | roleConfig: 12 | listenerClass: external-unstable # <1> 13 | ---- 14 | <1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` or a custom ListenerClass (the default setting is `cluster-internal`). 15 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/reference/commandline-parameters.adoc: -------------------------------------------------------------------------------- 1 | = Command line parameters 2 | 3 | This operator accepts the following command line parameters: 4 | 5 | == product-config 6 | 7 | *Default value*: `/etc/stackable/nifi-operator/config-spec/properties.yaml` 8 | 9 | *Required*: false 10 | 11 | *Multiple values:* false 12 | 13 | [source] 14 | ---- 15 | stackable-nifi-operator run --product-config /foo/bar/properties.yaml 16 | ---- 17 | 18 | == watch-namespace 19 | 20 | *Default value*: All namespaces 21 | 22 | *Required*: false 23 | 24 | *Multiple values:* false 25 | 26 | The operator **only** watches for resources in the provided namespace `test`: 27 | 28 | [source] 29 | ---- 30 | stackable-nifi-operator run --watch-namespace test 31 | ---- 32 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/00-rbac.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | kind: Role 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: test-role 6 | rules: 7 | {% if test_scenario['values']['openshift'] == "true" %} 8 | - apiGroups: ["security.openshift.io"] 9 | resources: ["securitycontextconstraints"] 10 | resourceNames: ["privileged"] 11 | verbs: ["use"] 12 | {% endif %} 13 | --- 14 | apiVersion: v1 15 | kind: ServiceAccount 16 | metadata: 17 | name: test-sa 18 | --- 19 | kind: RoleBinding 20 | apiVersion: rbac.authorization.k8s.io/v1 21 | metadata: 22 | name: test-rb 23 | subjects: 24 | - kind: ServiceAccount 25 | name: test-sa 26 | roleRef: 27 | kind: Role 28 | name: test-role 29 | apiGroup: rbac.authorization.k8s.io 30 | -------------------------------------------------------------------------------- /deploy/helm/nifi-operator/templates/_maintenance.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Create a list of maintenance related env vars. 3 | */}} 4 | {{- define "maintenance.envVars" -}} 5 | {{- with .Values.maintenance }} 6 | {{- if not .endOfSupportCheck.enabled }} 7 | - name: EOS_DISABLED 8 | value: "true" 9 | {{- end }} 10 | {{- if and .endOfSupportCheck.enabled .endOfSupportCheck.mode }} 11 | - name: EOS_CHECK_MODE 12 | value: {{ .endOfSupportCheck.mode }} 13 | {{ end }} 14 | {{- if and .endOfSupportCheck.enabled .endOfSupportCheck.interval }} 15 | - name: EOS_INTERVAL 16 | value: {{ .endOfSupportCheck.interval }} 17 | {{ end }} 18 | {{- if not .customResourceDefinitions.maintain }} 19 | - name: DISABLE_CRD_MAINTENANCE 20 | value: "true" 21 | {{- end }} 22 | {{- end }} 23 | {{- end }} 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/15_authentication-class.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: authentication.stackable.tech/v1alpha1 3 | kind: AuthenticationClass 4 | metadata: 5 | name: nifi-oidc-auth-class-$NAMESPACE 6 | spec: 7 | provider: 8 | oidc: 9 | hostname: keycloak.$NAMESPACE.svc.cluster.local 10 | rootPath: /realms/test/ 11 | principalClaim: preferred_username 12 | scopes: 13 | - openid 14 | - email 15 | - profile 16 | {% if test_scenario['values']['oidc-use-tls'] == 'true' %} 17 | port: 8443 18 | tls: 19 | verification: 20 | server: 21 | caCert: 22 | secretClass: keycloak-tls-$NAMESPACE 23 | {% else %} 24 | port: 8080 25 | tls: null 26 | {% endif %} 27 | -------------------------------------------------------------------------------- /nix/README.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | # Updating nix dependencies 7 | 8 | ## Run the following for an operator 9 | 10 | > [!NOTE] 11 | > We track the `master` branch of crate2nix as that is relatively up to date, but the releases are infrequent. 12 | 13 | ```shell 14 | niv update crate2nix 15 | niv update nixpkgs 16 | niv update beku.py -b X.Y.Z # Using the release tag 17 | ``` 18 | 19 | ### Test 20 | 21 | - Run make `regenerate-nix` to ensure crate2nix works 22 | - Run a smoke test to ensure beku.py works. 23 | - Run `make run-dev` to ensure nixpkgs are fine. 24 | 25 | ## Update operator-templating 26 | 27 | Do the same as above, but from `template/` 28 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/documentation.md.j2: -------------------------------------------------------------------------------- 1 | 2 | ## Documentation 3 | 4 | The stable documentation for this operator can be found in our [Stackable Data Platform documentation](https://docs.stackable.tech/home/stable/{{operator_docs_slug}}). 5 | If you are interested in the most recent state of this repository, check out the [nightly docs](https://docs.stackable.tech/home/nightly/{{operator_docs_slug}}) instead. 6 | 7 | The documentation for all Stackable products can be found at [docs.stackable.tech](https://docs.stackable.tech). 8 | 9 | If you have a question about the Stackable Data Platform, contact us via our [homepage](https://stackable.tech/) or ask a public question in our [Discussions forum](https://github.com/orgs/stackabletech/discussions). 10 | -------------------------------------------------------------------------------- /.readme/partials/main.md.j2: -------------------------------------------------------------------------------- 1 | This is a Kubernetes operator to manage [Apache NiFi](https://nifi.apache.org/) clusters. 2 | 3 | {% filter trim %} 4 | {%- include "partials/borrowed/overview_blurb.md.j2" -%} 5 | {% endfilter %} 6 | 7 | ## Installation 8 | 9 | You can install the operator using [stackablectl or helm](https://docs.stackable.tech/home/stable/{{operator_name}}/getting_started/installation). 10 | 11 | Read on to get started with it, or see it in action in one of our [demos](https://stackable.tech/en/demos/). 12 | 13 | ## Getting Started 14 | 15 | You can follow this [tutorial](https://docs.stackable.tech/home/stable/{{operator_name}}/getting_started/first_steps) . 16 | 17 | {% filter trim %} 18 | {%- include "partials/borrowed/documentation.md.j2" -%} 19 | {% endfilter %} 20 | -------------------------------------------------------------------------------- /tests/release.yaml: -------------------------------------------------------------------------------- 1 | # Contains all operators required to run the test suite. 2 | --- 3 | releases: 4 | # Do not change the name of the release as it's referenced from run-tests 5 | tests: 6 | releaseDate: 1970-01-01 7 | description: Integration test 8 | products: 9 | commons: 10 | operatorVersion: 0.0.0-dev 11 | secret: 12 | operatorVersion: 0.0.0-dev 13 | listener: 14 | operatorVersion: 0.0.0-dev 15 | opa: 16 | operatorVersion: 0.0.0-dev 17 | zookeeper: 18 | operatorVersion: 0.0.0-dev 19 | hdfs: 20 | operatorVersion: 0.0.0-dev 21 | hive: 22 | operatorVersion: 0.0.0-dev 23 | trino: 24 | operatorVersion: 0.0.0-dev 25 | nifi: 26 | operatorVersion: 0.0.0-dev 27 | -------------------------------------------------------------------------------- /.readme/partials/borrowed/links.md.j2: -------------------------------------------------------------------------------- 1 | 2 | [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/stackabletech/{{operator_name}}-operator/graphs/commit-activity) 3 | [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-green.svg)](https://docs.stackable.tech/home/stable/contributor/index.html) 4 | [![License OSL3.0](https://img.shields.io/badge/license-OSL3.0-green)](./LICENSE) 5 | 6 | [Documentation](https://docs.stackable.tech/home/stable/{{operator_docs_slug}}) {% if quickstart_link %}| [Quickstart]({{quickstart_link}}) {% endif %}| [Stackable Data Platform](https://stackable.tech/) | [Platform Docs](https://docs.stackable.tech/) | [Discussions](https://github.com/orgs/stackabletech/discussions) | [Discord](https://discord.gg/7kZ3BNnCAF) 7 | -------------------------------------------------------------------------------- /scripts/generate-manifests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This script reads a Helm chart from deploy/helm/nifi-operator and 3 | # generates manifest files into deploy/manifestss 4 | set -e 5 | 6 | tmp=$(mktemp -d ./manifests-XXXXX) 7 | 8 | helm template --output-dir "$tmp" \ 9 | --include-crds \ 10 | --name-template nifi-operator \ 11 | deploy/helm/nifi-operator 12 | 13 | for file in "$tmp"/nifi-operator/*/*; do 14 | yq eval -i 'del(.. | select(has("app.kubernetes.io/managed-by")) | ."app.kubernetes.io/managed-by")' /dev/stdin < "$file" 15 | yq eval -i 'del(.. | select(has("helm.sh/chart")) | ."helm.sh/chart")' /dev/stdin < "$file" 16 | sed -i '/# Source: .*/d' "$file" 17 | done 18 | 19 | cp -r "$tmp"/nifi-operator/*/* deploy/manifests/ 20 | 21 | rm -rf "$tmp" 22 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/20-install-zookeeper.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if test_scenario['values']['use-zookeeper-manager'] == 'true' %} 2 | --- 3 | apiVersion: zookeeper.stackable.tech/v1alpha1 4 | kind: ZookeeperCluster 5 | metadata: 6 | name: zookeeper 7 | spec: 8 | image: 9 | productVersion: "{{ test_scenario['values']['zookeeper'] }}" 10 | pullPolicy: IfNotPresent 11 | clusterConfig: 12 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 13 | vectorAggregatorConfigMapName: vector-aggregator-discovery 14 | {% endif %} 15 | servers: 16 | config: 17 | logging: 18 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 19 | roleConfig: 20 | listenerClass: {{ test_scenario['values']['listener-class'] }} 21 | roleGroups: 22 | default: 23 | replicas: 1 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/10-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: test-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: test-nifi-znode 26 | spec: 27 | clusterRef: 28 | name: test-zk 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/20-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: test-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | clusterConfig: 11 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: test-nifi-znode 26 | spec: 27 | clusterRef: 28 | name: test-zk 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/20-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: test-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | clusterConfig: 11 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | logging: 17 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 18 | roleGroups: 19 | default: 20 | replicas: 1 21 | --- 22 | apiVersion: zookeeper.stackable.tech/v1alpha1 23 | kind: ZookeeperZnode 24 | metadata: 25 | name: test-nifi-znode 26 | spec: 27 | clusterRef: 28 | name: test-zk 29 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/10-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: test-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | gracefulShutdownTimeout: 1m 17 | logging: 18 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 19 | roleGroups: 20 | default: 21 | replicas: 1 22 | --- 23 | apiVersion: zookeeper.stackable.tech/v1alpha1 24 | kind: ZookeeperZnode 25 | metadata: 26 | name: nifi-znode 27 | spec: 28 | clusterRef: 29 | name: test-zk 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/10-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: test-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | gracefulShutdownTimeout: 1m 17 | logging: 18 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 19 | roleGroups: 20 | default: 21 | replicas: 1 22 | --- 23 | apiVersion: zookeeper.stackable.tech/v1alpha1 24 | kind: ZookeeperZnode 25 | metadata: 26 | name: nifi-with-ldap-znode 27 | spec: 28 | clusterRef: 29 | name: test-zk 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/01-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: test-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | gracefulShutdownTimeout: 1m 17 | logging: 18 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 19 | roleGroups: 20 | default: 21 | replicas: 1 22 | --- 23 | apiVersion: zookeeper.stackable.tech/v1alpha1 24 | kind: ZookeeperZnode 25 | metadata: 26 | name: test-nifi-znode 27 | spec: 28 | clusterRef: 29 | name: test-zk 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/01-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: test-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | gracefulShutdownTimeout: 1m 17 | logging: 18 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 19 | roleGroups: 20 | default: 21 | replicas: 1 22 | --- 23 | apiVersion: zookeeper.stackable.tech/v1alpha1 24 | kind: ZookeeperZnode 25 | metadata: 26 | name: test-nifi-znode 27 | spec: 28 | clusterRef: 29 | name: test-zk 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/01-normal-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Normal issue 3 | about: This is just a normal empty issue with a simple checklist 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Issue checklist 11 | 12 | This is a simple checklist of things to bear in mind when creating a new issue. 13 | 14 | - [ ] **Describe the use-case**: As far as possible, use the pattern "As a [type of user], I would like [feature/functionality] to be able to do [specific action]." This helps identify the feature and the problem it addresses. 15 | - [ ] **Indicate importance and urgency**: Use a scale (e.g., low, medium, high) to indicate the level of importance and urgency. 16 | - [ ] **Work-around**: If there is a known work-around, describe it briefly. 17 | - [ ] **Environment**: Describe the environment where the issue occurs (e.g., SDP version, K8S version, etc.). 18 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/01_s3-connection.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: s3.stackable.tech/v1alpha1 3 | kind: S3Connection 4 | metadata: 5 | name: minio 6 | spec: 7 | host: "minio.${NAMESPACE}.svc.cluster.local" 8 | port: 9000 9 | accessStyle: Path 10 | credentials: 11 | secretClass: s3-credentials-class 12 | tls: 13 | verification: 14 | server: 15 | caCert: 16 | secretClass: tls 17 | --- 18 | apiVersion: secrets.stackable.tech/v1alpha1 19 | kind: SecretClass 20 | metadata: 21 | name: s3-credentials-class 22 | spec: 23 | backend: 24 | k8sSearch: 25 | searchNamespace: 26 | pod: {} 27 | --- 28 | apiVersion: v1 29 | kind: Secret 30 | metadata: 31 | name: minio-credentials 32 | labels: 33 | secrets.stackable.tech/class: s3-credentials-class 34 | stringData: 35 | accessKey: admin 36 | secretKey: adminadmin 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/02-install-zookeeper.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: test-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | gracefulShutdownTimeout: 1m 17 | logging: 18 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 19 | roleGroups: 20 | default: 21 | replicas: 1 22 | --- 23 | apiVersion: zookeeper.stackable.tech/v1alpha1 24 | kind: ZookeeperZnode 25 | metadata: 26 | name: test-nifi-znode 27 | spec: 28 | clusterRef: 29 | name: test-zk 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned_resources/01-install-zk.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: zookeeper.stackable.tech/v1alpha1 3 | kind: ZookeeperCluster 4 | metadata: 5 | name: test-zk 6 | spec: 7 | image: 8 | productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" 9 | pullPolicy: IfNotPresent 10 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 11 | clusterConfig: 12 | vectorAggregatorConfigMapName: vector-aggregator-discovery 13 | {% endif %} 14 | servers: 15 | config: 16 | gracefulShutdownTimeout: 1m 17 | logging: 18 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 19 | roleGroups: 20 | default: 21 | replicas: 1 22 | --- 23 | apiVersion: zookeeper.stackable.tech/v1alpha1 24 | kind: ZookeeperZnode 25 | metadata: 26 | name: test-nifi-znode 27 | spec: 28 | clusterRef: 29 | name: test-zk 30 | -------------------------------------------------------------------------------- /rust/operator-binary/src/crd/tls.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use stackable_operator::schemars::{self, JsonSchema}; 3 | 4 | #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] 5 | #[serde(rename_all = "camelCase")] 6 | pub struct NifiTls { 7 | /// This only affects client connections and is used to 8 | /// control which certificate the servers should use to 9 | /// authenticate themselves against the client. 10 | #[serde(default = "NifiTls::default_server_secret_class")] 11 | pub server_secret_class: String, 12 | } 13 | 14 | impl Default for NifiTls { 15 | fn default() -> Self { 16 | Self { 17 | server_secret_class: Self::default_server_secret_class(), 18 | } 19 | } 20 | } 21 | 22 | impl NifiTls { 23 | fn default_server_secret_class() -> String { 24 | "tls".to_owned() 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # All defaults or options can be checked here: 3 | # https://github.com/DavidAnson/markdownlint/blob/main/schema/.markdownlint.yaml 4 | 5 | # Default state for all rules 6 | default: true 7 | 8 | # MD013/line-length - Line length 9 | MD013: 10 | # Number of characters 11 | line_length: 9999 12 | # Number of characters for headings 13 | heading_line_length: 9999 14 | # Number of characters for code blocks 15 | code_block_line_length: 9999 16 | 17 | # MD033/no-inline-html 18 | MD033: 19 | allowed_elements: [h1, img, p] 20 | 21 | # MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the same content 22 | MD024: 23 | # Only check sibling headings 24 | siblings_only: true 25 | 26 | # MD041/first-line-heading/first-line-h1 First line in a file should be a top-level heading 27 | MD041: false # Github issues and PRs already have titles, and H1 is enormous in the description box. 28 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/extra-volumes.adoc: -------------------------------------------------------------------------------- 1 | = Adding external files to the NiFi servers 2 | :description: Add external files to NiFi Pods by specifying extra volumes, such as client certificates or keytabs. 3 | 4 | Since Apache NiFi allows executing arbitrary workflows depending on which processors are used, it may become necessary to add external files to the Pods. 5 | These could for example be client certificates used to configure a `PollHTTP` processor, a keytab to obtain a Kerberos ticket, or similar things. 6 | 7 | In order to make these files available the operator allows specifying extra volumes that are added to the NiFi Pods. 8 | 9 | [source,yaml] 10 | ---- 11 | spec: 12 | clusterConfig: 13 | extraVolumes: 14 | - name: nifi-client-certs 15 | secret: 16 | secretName: nifi-client-certs 17 | ---- 18 | 19 | All Volumes specified in this section are made available under `/stackable/userdata/\{volumename\}`. 20 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/getting_started/index.adoc: -------------------------------------------------------------------------------- 1 | = Getting started 2 | :description: Get started with Apache NiFi using the Stackable operator: install, set up your cluster, and configure resources. 3 | 4 | This guide gets you started with Apache NiFi using the Stackable operator. 5 | It guides you through the installation of the operator and its dependencies, setting up your first NiFi cluster. 6 | 7 | == Prerequisites 8 | 9 | You need: 10 | 11 | * a Kubernetes cluster 12 | * kubectl 13 | * optional: Helm 14 | 15 | Resource sizing depends on cluster type(s), usage and scope, but as a starting point we recommend a minimum of the following resources for this operator: 16 | 17 | * 0.2 cores (e.g. i5 or similar) 18 | * 256MB RAM 19 | 20 | == What's next 21 | 22 | The guide is divided into the following steps: 23 | 24 | * xref:getting_started/installation.adoc[Installing the Operators] 25 | * xref:getting_started/first_steps.adoc[Setting up the NiFi cluster] 26 | -------------------------------------------------------------------------------- /deploy/helm/nifi-operator/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Helm Chart for Stackable Operator for Apache NiFi 3 | 4 | This Helm Chart can be used to install Custom Resource Definitions and the Operator for Apache NiFi provided by Stackable. 5 | 6 | ## Requirements 7 | 8 | - Create a [Kubernetes Cluster](../Readme.md) 9 | - Install [Helm](https://helm.sh/docs/intro/install/) 10 | 11 | ## Install the Stackable Operator for Apache NiFi 12 | 13 | ```bash 14 | # From the root of the operator repository 15 | make compile-chart 16 | 17 | helm install nifi-operator deploy/helm/nifi-operator 18 | ``` 19 | 20 | ## Usage of the CRDs 21 | 22 | The usage of this operator and its CRDs is described in the [documentation](https://docs.stackable.tech/nifi/index.html) 23 | 24 | The operator has example requests included in the [`/examples`](https://github.com/stackabletech/nifi-operator/tree/main/examples) directory. 25 | 26 | ## Links 27 | 28 | 29 | -------------------------------------------------------------------------------- /rust/operator-binary/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "stackable-nifi-operator" 3 | description = "Stackable Operator for Apache NiFi" 4 | version.workspace = true 5 | authors.workspace = true 6 | license.workspace = true 7 | edition.workspace = true 8 | repository.workspace = true 9 | publish = false 10 | 11 | [dependencies] 12 | product-config.workspace = true 13 | stackable-operator.workspace = true 14 | 15 | anyhow.workspace = true 16 | clap.workspace = true 17 | const_format.workspace = true 18 | fnv.workspace = true 19 | futures.workspace = true 20 | indoc.workspace = true 21 | pin-project.workspace = true 22 | rand.workspace = true 23 | semver.workspace = true 24 | serde.workspace = true 25 | serde_json.workspace = true 26 | snafu.workspace = true 27 | strum.workspace = true 28 | tokio.workspace = true 29 | tracing.workspace = true 30 | url.workspace = true 31 | 32 | [dev-dependencies] 33 | rstest.workspace = true 34 | serde_yaml.workspace = true 35 | 36 | [build-dependencies] 37 | built.workspace = true 38 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/25_helm-bitnami-postgresql-values.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | security: 4 | allowInsecureImages: true # needed starting with Chart version 16.3.0 if modifying images 5 | 6 | image: 7 | repository: bitnamilegacy/postgresql 8 | 9 | volumePermissions: 10 | enabled: false 11 | image: 12 | repository: bitnamilegacy/os-shell 13 | securityContext: 14 | runAsUser: auto 15 | 16 | metrics: 17 | image: 18 | repository: bitnamilegacy/postgres-exporter 19 | 20 | primary: 21 | extendedConfiguration: | 22 | password_encryption=md5 23 | podSecurityContext: 24 | {% if test_scenario['values']['openshift'] == 'true' %} 25 | enabled: false 26 | {% else %} 27 | enabled: true 28 | {% endif %} 29 | containerSecurityContext: 30 | enabled: false 31 | resources: 32 | requests: 33 | memory: "512Mi" 34 | cpu: "512m" 35 | limits: 36 | memory: "512Mi" 37 | cpu: "1" 38 | 39 | auth: 40 | username: hive 41 | password: hive 42 | database: hive 43 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/create-authentication-classes.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: authentication.stackable.tech/v1alpha1 3 | kind: AuthenticationClass 4 | metadata: 5 | {% if test_scenario['values']['ldap-use-tls'] == 'false' %} 6 | name: ldap-without-tls 7 | {% else %} 8 | name: ldap-with-tls 9 | {% endif %} 10 | spec: 11 | provider: 12 | ldap: 13 | hostname: openldap.$NAMESPACE.svc.cluster.local 14 | searchBase: ou=my users,dc=example,dc=org 15 | bindCredentials: 16 | secretClass: nifi-with-ldap-bind 17 | {% if test_scenario['values']['ldap-use-tls'] == 'false' %} 18 | port: 1389 19 | {% else %} 20 | port: 1636 21 | tls: 22 | verification: 23 | server: 24 | caCert: 25 | secretClass: openldap-tls 26 | {% endif %} 27 | --- 28 | apiVersion: secrets.stackable.tech/v1alpha1 29 | kind: SecretClass 30 | metadata: 31 | name: nifi-with-ldap-bind 32 | spec: 33 | backend: 34 | k8sSearch: 35 | searchNamespace: 36 | pod: {} 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | name: create-iceberg-tables 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: create-iceberg-tables 11 | image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-l'] }}-stackable0.0.0-dev" 12 | command: 13 | - bash 14 | - -euo 15 | - pipefail 16 | - -c 17 | - | 18 | cat << 'EOF' | ./trino-cli --server https://trino-coordinator:8443 --insecure --user admin 19 | CREATE SCHEMA IF NOT EXISTS iceberg.s3 WITH (location = 's3a://demo/lakehouse/s3'); 20 | CREATE TABLE IF NOT EXISTS iceberg.s3.greetings (hello varchar); 21 | CREATE SCHEMA IF NOT EXISTS iceberg.hdfs WITH (location = 'hdfs:/lakehouse/hdfs'); 22 | CREATE TABLE IF NOT EXISTS iceberg.hdfs.greetings (hello varchar); 23 | EOF 24 | restartPolicy: OnFailure 25 | -------------------------------------------------------------------------------- /deploy/helm/nifi-operator/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | {{ if .Values.serviceAccount.create -}} 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: {{ include "operator.fullname" . }}-serviceaccount 7 | labels: 8 | {{- include "operator.labels" . | nindent 4 }} 9 | {{- with .Values.serviceAccount.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | # This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. 16 | kind: ClusterRoleBinding 17 | metadata: 18 | name: {{ include "operator.fullname" . }}-clusterrolebinding 19 | labels: 20 | {{- include "operator.labels" . | nindent 4 }} 21 | subjects: 22 | - kind: ServiceAccount 23 | name: {{ include "operator.fullname" . }}-serviceaccount 24 | namespace: {{ .Release.Namespace }} 25 | roleRef: 26 | kind: ClusterRole 27 | name: {{ include "operator.fullname" . }}-clusterrole 28 | apiGroup: rbac.authorization.k8s.io 29 | {{- end }} 30 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/30-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: install-nifi 6 | timeout: 1200 7 | commands: 8 | - script: kubectl -n $NAMESPACE wait --for=condition=available=true nificlusters.nifi.stackable.tech/test-nifi --timeout 1201s 9 | --- 10 | apiVersion: apps/v1 11 | kind: StatefulSet 12 | metadata: 13 | name: test-nifi-node-default 14 | status: 15 | readyReplicas: 2 16 | replicas: 2 17 | --- 18 | apiVersion: policy/v1 19 | kind: PodDisruptionBudget 20 | metadata: 21 | name: test-nifi-node 22 | status: 23 | expectedPods: 2 24 | currentHealthy: 2 25 | disruptionsAllowed: 1 26 | --- 27 | apiVersion: v1 28 | kind: Service 29 | metadata: 30 | name: test-nifi-node 31 | spec: 32 | type: NodePort # external-unstable 33 | --- 34 | apiVersion: v1 35 | kind: Service 36 | metadata: 37 | name: test-nifi-node-default-headless 38 | spec: 39 | type: ClusterIP 40 | --- 41 | apiVersion: v1 42 | kind: Service 43 | metadata: 44 | name: test-nifi-node-default-metrics 45 | spec: 46 | type: ClusterIP 47 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/07-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | metadata: 5 | name: test-nifi-updated 6 | timeout: 300 7 | commands: 8 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi.py -u admin -p supersecretpassword -n $NAMESPACE -c 3 9 | {% if test_scenario['values']['nifi_new'].split(',')[0].startswith('1.') %} 10 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE 11 | {% endif %} 12 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-headless.$NAMESPACE.svc.cluster.local:8443 query > /tmp/new_input" 13 | # This tests if the output contains an Error or zero flow files are queued, which also indicates that something went wrong 14 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "cat /tmp/new_input | grep -Eov 'Error|\b0\b'" 15 | # This tests that the number of input records stays the same after the upgrade. 16 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- diff /tmp/old_input /tmp/new_input 17 | -------------------------------------------------------------------------------- /scripts/docs_templating.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Reads a file with variables to insert into templates, and templates all .*.j2 files 5 | # in the 'docs' directory. 6 | # 7 | # dependencies 8 | # pip install jinja2-cli 9 | 10 | docs_dir="$(dirname "$0")/../docs" 11 | templating_vars_file="$docs_dir/templating_vars.yaml" 12 | 13 | # Check if files need templating 14 | if [[ -z $(find "$docs_dir" -name '*.j2') ]]; 15 | then 16 | echo "No files need templating, exiting." 17 | exit 18 | fi 19 | 20 | # Check if jinja2 is there 21 | if ! command -v jinja2 &> /dev/null 22 | then 23 | echo "jinja2 could not be found. Use 'pip install jinja2-cli' to install it." 24 | exit 1 25 | fi 26 | 27 | # Check if templating vars file exists 28 | if [[ ! -f "$templating_vars_file" ]]; 29 | then 30 | echo "$templating_vars_file does not exist, cannot start templating." 31 | fi 32 | 33 | find "$docs_dir" -name '*.j2' | 34 | while read -r file 35 | do 36 | new_file_name=${file%.j2} # Remove .j2 suffix 37 | echo "templating $new_file_name" 38 | jinja2 "$file" "$templating_vars_file" -o "$new_file_name" 39 | done 40 | 41 | echo "done" 42 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/new_version.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: New Version 3 | about: Request support for a new product version 4 | title: "[NEW VERSION]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Which new version of Apache NiFi should we support? 11 | 12 | Please specify the version, version range or version numbers to support, please also add these to the issue title 13 | 14 | ## Additional information 15 | 16 | If possible, provide a link to release notes/changelog 17 | 18 | ## Changes required 19 | 20 | Are there any upstream changes that we need to support? 21 | e.g. new features, changed features, deprecated features etc. 22 | 23 | ## Implementation checklist 24 | 25 | 29 | 30 | - [ ] Update the Docker image 31 | - [ ] Update documentation to include supported version(s) 32 | - [ ] Update and test getting started guide with updated version(s) 33 | - [ ] Update operator to support the new version (if needed) 34 | - [ ] Update integration tests to test use the new versions (in addition or replacing old versions 35 | - [ ] Update examples to use new versions 36 | -------------------------------------------------------------------------------- /tests/kuttl-test.yaml.jinja2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestSuite 4 | testDirs: 5 | {% for testcase in testinput.tests %} 6 | - ./tests/{{ testcase.name }} 7 | {% endfor %} 8 | 9 | startKIND: false 10 | suppress: ["events"] 11 | parallel: 2 12 | 13 | # The timeout (in seconds) is used when namespaces are created or 14 | # deleted, and, if not overridden, in TestSteps, TestAsserts, and 15 | # Commands. If not set, the timeout is 30 seconds by default. 16 | # 17 | # The deletion of a namespace can take a while until all resources, 18 | # especially PersistentVolumeClaims, are gracefully shut down. If the 19 | # timeout is reached in the meantime, even a successful test case is 20 | # considered a failure. 21 | # 22 | # For instance, the termination grace period of the Vector aggregator in 23 | # the logging tests is set to 60 seconds. If there are logs entries 24 | # which could not be forwarded yet to the external aggregator defined in 25 | # the VECTOR_AGGREGATOR environment variable, then the test aggregator 26 | # uses this period of time by trying to forward the events. In this 27 | # case, deleting a namespace with several Pods takes about 90 seconds. 28 | timeout: 300 29 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pre-release-getting-started-script.md: -------------------------------------------------------------------------------- 1 | ## Check and Update Getting Started Script 2 | 3 | 7 | 8 | 11 | 12 | Part of 13 | 14 | > [!NOTE] 15 | > During a Stackable release we need to check (and optionally update) the 16 | > getting-started scripts to ensure they still work after product and operator 17 | > updates. 18 | 19 | ```shell 20 | # Some of the scripts are in a code/ subdirectory 21 | # pushd docs/modules/superset/examples/getting_started 22 | # pushd docs/modules/superset/examples/getting_started/code 23 | pushd $(fd -td getting_started | grep examples); cd code 2>/dev/null || true 24 | 25 | # Make a fresh cluster (~12 seconds) 26 | kind delete cluster && kind create cluster 27 | ./getting_started.sh stackablectl 28 | 29 | # Make a fresh cluster (~12 seconds) 30 | kind delete cluster && kind create cluster 31 | ./getting_started.sh helm 32 | 33 | popd 34 | ``` 35 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/30-stop-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: nifi.stackable.tech/v1alpha1 3 | kind: NifiCluster 4 | metadata: 5 | name: test-nifi 6 | spec: 7 | image: 8 | {% if test_scenario['values']['nifi-latest'].find(",") > 0 %} 9 | custom: "{{ test_scenario['values']['nifi-latest'].split(',')[1] }}" 10 | productVersion: "{{ test_scenario['values']['nifi-latest'].split(',')[0] }}" 11 | {% else %} 12 | custom: null 13 | productVersion: "{{ test_scenario['values']['nifi-latest'] }}" 14 | {% endif %} 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | authentication: 18 | - authenticationClass: simple-nifi-users 19 | sensitiveProperties: 20 | keySecret: nifi-sensitive-property-key 21 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 22 | vectorAggregatorConfigMapName: vector-aggregator-discovery 23 | {% endif %} 24 | zookeeperConfigMapName: test-nifi-znode 25 | clusterOperation: 26 | stopped: true 27 | reconciliationPaused: false 28 | nodes: 29 | config: 30 | gracefulShutdownTimeout: 1m 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 2 36 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/40-pause-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: nifi.stackable.tech/v1alpha1 3 | kind: NifiCluster 4 | metadata: 5 | name: test-nifi 6 | spec: 7 | image: 8 | {% if test_scenario['values']['nifi-latest'].find(",") > 0 %} 9 | custom: "{{ test_scenario['values']['nifi-latest'].split(',')[1] }}" 10 | productVersion: "{{ test_scenario['values']['nifi-latest'].split(',')[0] }}" 11 | {% else %} 12 | custom: null 13 | productVersion: "{{ test_scenario['values']['nifi-latest'] }}" 14 | {% endif %} 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | authentication: 18 | - authenticationClass: simple-nifi-users 19 | sensitiveProperties: 20 | keySecret: nifi-sensitive-property-key 21 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 22 | vectorAggregatorConfigMapName: vector-aggregator-discovery 23 | {% endif %} 24 | zookeeperConfigMapName: test-nifi-znode 25 | clusterOperation: 26 | stopped: false 27 | reconciliationPaused: true 28 | nodes: 29 | config: 30 | gracefulShutdownTimeout: 1m 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 2 36 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/50-restart-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: nifi.stackable.tech/v1alpha1 3 | kind: NifiCluster 4 | metadata: 5 | name: test-nifi 6 | spec: 7 | image: 8 | {% if test_scenario['values']['nifi-latest'].find(",") > 0 %} 9 | custom: "{{ test_scenario['values']['nifi-latest'].split(',')[1] }}" 10 | productVersion: "{{ test_scenario['values']['nifi-latest'].split(',')[0] }}" 11 | {% else %} 12 | custom: null 13 | productVersion: "{{ test_scenario['values']['nifi-latest'] }}" 14 | {% endif %} 15 | pullPolicy: IfNotPresent 16 | clusterConfig: 17 | authentication: 18 | - authenticationClass: simple-nifi-users 19 | sensitiveProperties: 20 | keySecret: nifi-sensitive-property-key 21 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 22 | vectorAggregatorConfigMapName: vector-aggregator-discovery 23 | {% endif %} 24 | zookeeperConfigMapName: test-nifi-znode 25 | clusterOperation: 26 | stopped: false 27 | reconciliationPaused: false 28 | nodes: 29 | config: 30 | gracefulShutdownTimeout: 1m 31 | logging: 32 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 33 | roleGroups: 34 | default: 35 | replicas: 2 36 | -------------------------------------------------------------------------------- /deploy/config-spec/properties.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | spec: 3 | units: [] 4 | 5 | properties: 6 | - property: 7 | propertyNames: 8 | - name: "networkaddress.cache.ttl" 9 | kind: 10 | type: "file" 11 | file: "security.properties" 12 | datatype: 13 | type: "integer" 14 | min: "0" 15 | recommendedValues: 16 | - fromVersion: "0.0.0" 17 | value: "30" 18 | roles: 19 | - name: "node" 20 | required: true 21 | asOfVersion: "0.0.0" 22 | comment: "TTL for successfully resolved domain names." 23 | description: "TTL for successfully resolved domain names." 24 | 25 | - property: 26 | propertyNames: 27 | - name: "networkaddress.cache.negative.ttl" 28 | kind: 29 | type: "file" 30 | file: "security.properties" 31 | datatype: 32 | type: "integer" 33 | min: "0" 34 | recommendedValues: 35 | - fromVersion: "0.0.0" 36 | value: "0" 37 | roles: 38 | - name: "node" 39 | required: true 40 | asOfVersion: "0.0.0" 41 | comment: "TTL for host names that cannot be resolved." 42 | description: "TTL for host names that cannot be resolved." 43 | -------------------------------------------------------------------------------- /deploy/helm/nifi-operator/configs/properties.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | spec: 3 | units: [] 4 | 5 | properties: 6 | - property: 7 | propertyNames: 8 | - name: "networkaddress.cache.ttl" 9 | kind: 10 | type: "file" 11 | file: "security.properties" 12 | datatype: 13 | type: "integer" 14 | min: "0" 15 | recommendedValues: 16 | - fromVersion: "0.0.0" 17 | value: "30" 18 | roles: 19 | - name: "node" 20 | required: true 21 | asOfVersion: "0.0.0" 22 | comment: "TTL for successfully resolved domain names." 23 | description: "TTL for successfully resolved domain names." 24 | 25 | - property: 26 | propertyNames: 27 | - name: "networkaddress.cache.negative.ttl" 28 | kind: 29 | type: "file" 30 | file: "security.properties" 31 | datatype: 32 | type: "integer" 33 | min: "0" 34 | recommendedValues: 35 | - fromVersion: "0.0.0" 36 | value: "0" 37 | roles: 38 | - name: "node" 39 | required: true 40 | asOfVersion: "0.0.0" 41 | comment: "TTL for host names that cannot be resolved." 42 | description: "TTL for host names that cannot be resolved." 43 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/04-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 300 5 | commands: 6 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi.py -u admin -p supersecretpassword -n $NAMESPACE -c 3 7 | {% if test_scenario['values']['nifi_old'].split(',')[0].startswith('1.') %} 8 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE 9 | {% endif %} 10 | {% if test_scenario['values']['nifi_old'].split(',')[0] == '2.0.0' %} 11 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-headless.$NAMESPACE.svc.cluster.local:8443 run json /tmp/generate-and-log-flowfiles.json > /tmp/old_input" 12 | {% else %} 13 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-headless.$NAMESPACE.svc.cluster.local:8443 run template /tmp/generate-and-log-flowfiles.xml > /tmp/old_input" 14 | {% endif %} 15 | # This tests if the output contains an Error or zero flow files are queued, which also indicates that something went wrong 16 | - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "cat /tmp/old_input | grep -Eov 'Error|\b0\b'" 17 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/32-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | commands: 6 | # 7 | # Test envOverrides 8 | # 9 | - script: | 10 | kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' 11 | kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' 12 | kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' 13 | # 14 | # Test configOverrides 15 | # 16 | - script: | 17 | kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.enabled=false" 18 | kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.verbose=false" 19 | kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.max.filecount=20" 20 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/32-assert.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | timeout: 600 5 | commands: 6 | # 7 | # Test envOverrides 8 | # 9 | - script: | 10 | kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' 11 | kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' 12 | kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' 13 | # 14 | # Test configOverrides 15 | # 16 | - script: | 17 | kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.enabled=false" 18 | kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.verbose=false" 19 | kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.max.filecount=20" 20 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/cacert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDVTCCAj2gAwIBAgIJAJ8/0entaUgnMA0GCSqGSIb3DQEBCwUAMCYxJDAiBgNV 3 | BAMMG3NlY3JldC1vcGVyYXRvciBzZWxmLXNpZ25lZDAeFw0yMjAxMTIxNDU3NDVa 4 | Fw0yNDAxMTIxNTAyNDVaMCYxJDAiBgNVBAMMG3NlY3JldC1vcGVyYXRvciBzZWxm 5 | LXNpZ25lZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALLKNGhq5gE+ 6 | mL9zFCLqtc22CLk8BSbjesjUEhBK3kxDvFDa2ou5atH0eUFjtOSszay2oBrCTVWK 7 | wZBsdUkL0HkW/wq9A8EUkQ8EownXnsxpI61CLNGLPpBZc+CRHhyWDD6BqwGvEHEv 8 | W546mh6k49//7zCiYfTK9/LCKBCFdDV6Sb7mNJ8HbNUj54uwC6iOgH25OCRDh4Bt 9 | zXoSrV9GLAm6AM25ZFo+ONOUBMtv7pavaR0CFMnAixl2NKV2wyLBYAYnJgdJFzGD 10 | 8mP6HwuR7e2g7PkcyC01EnX4iOIuuKHT/Xl9ynut4nHI7g6popotgashrQ5Jf8MS 11 | Kf98O12LzSMCAwEAAaOBhTCBgjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRf 12 | U9OxCBwCqYiUjWqY05sz3a6cmjBABgNVHSMEOTA3oSqkKDAmMSQwIgYDVQQDDBtz 13 | ZWNyZXQtb3BlcmF0b3Igc2VsZi1zaWduZWSCCQCfP9Hp7WlIJzAOBgNVHQ8BAf8E 14 | BAMCAYYwDQYJKoZIhvcNAQELBQADggEBAA8Flk1XOb1pH33Mbie5ronP2xw/xf6t 15 | Ox3PBEZ+5/jSPdIwoSaRp9JoP0L9Rg68jzcl5QMa4pOYWe+C1q8aZP0tjfq1eJfO 16 | UD5ik2DQgEuoF1ELgW1xoM38vkd8wgE711swDHK2zAsOudSzO4XZ4rQ6kaXXtoej 17 | 2kFhxDYcC+na90LdkJM0kAqrjxlFaP7WgUK+HA2iN00CFSOI9FVdppLtootbcb+y 18 | +WfXxM7gA9Exg4f2vKGVx7UxB/k4AbPvogBQZvK8VoAQocAhWrw7o2rqAesAw6JD 19 | WwQjM69TlEfbHYXtTfMbi01Wi5TtVhFCjyXK6KDsqSgU+9McExIy70k= 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/cacert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDVTCCAj2gAwIBAgIJAJ8/0entaUgnMA0GCSqGSIb3DQEBCwUAMCYxJDAiBgNV 3 | BAMMG3NlY3JldC1vcGVyYXRvciBzZWxmLXNpZ25lZDAeFw0yMjAxMTIxNDU3NDVa 4 | Fw0yNDAxMTIxNTAyNDVaMCYxJDAiBgNVBAMMG3NlY3JldC1vcGVyYXRvciBzZWxm 5 | LXNpZ25lZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALLKNGhq5gE+ 6 | mL9zFCLqtc22CLk8BSbjesjUEhBK3kxDvFDa2ou5atH0eUFjtOSszay2oBrCTVWK 7 | wZBsdUkL0HkW/wq9A8EUkQ8EownXnsxpI61CLNGLPpBZc+CRHhyWDD6BqwGvEHEv 8 | W546mh6k49//7zCiYfTK9/LCKBCFdDV6Sb7mNJ8HbNUj54uwC6iOgH25OCRDh4Bt 9 | zXoSrV9GLAm6AM25ZFo+ONOUBMtv7pavaR0CFMnAixl2NKV2wyLBYAYnJgdJFzGD 10 | 8mP6HwuR7e2g7PkcyC01EnX4iOIuuKHT/Xl9ynut4nHI7g6popotgashrQ5Jf8MS 11 | Kf98O12LzSMCAwEAAaOBhTCBgjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRf 12 | U9OxCBwCqYiUjWqY05sz3a6cmjBABgNVHSMEOTA3oSqkKDAmMSQwIgYDVQQDDBtz 13 | ZWNyZXQtb3BlcmF0b3Igc2VsZi1zaWduZWSCCQCfP9Hp7WlIJzAOBgNVHQ8BAf8E 14 | BAMCAYYwDQYJKoZIhvcNAQELBQADggEBAA8Flk1XOb1pH33Mbie5ronP2xw/xf6t 15 | Ox3PBEZ+5/jSPdIwoSaRp9JoP0L9Rg68jzcl5QMa4pOYWe+C1q8aZP0tjfq1eJfO 16 | UD5ik2DQgEuoF1ELgW1xoM38vkd8wgE711swDHK2zAsOudSzO4XZ4rQ6kaXXtoej 17 | 2kFhxDYcC+na90LdkJM0kAqrjxlFaP7WgUK+HA2iN00CFSOI9FVdppLtootbcb+y 18 | +WfXxM7gA9Exg4f2vKGVx7UxB/k4AbPvogBQZvK8VoAQocAhWrw7o2rqAesAw6JD 19 | WwQjM69TlEfbHYXtTfMbi01Wi5TtVhFCjyXK6KDsqSgU+9McExIy70k= 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/cacert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDVTCCAj2gAwIBAgIJAJ8/0entaUgnMA0GCSqGSIb3DQEBCwUAMCYxJDAiBgNV 3 | BAMMG3NlY3JldC1vcGVyYXRvciBzZWxmLXNpZ25lZDAeFw0yMjAxMTIxNDU3NDVa 4 | Fw0yNDAxMTIxNTAyNDVaMCYxJDAiBgNVBAMMG3NlY3JldC1vcGVyYXRvciBzZWxm 5 | LXNpZ25lZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALLKNGhq5gE+ 6 | mL9zFCLqtc22CLk8BSbjesjUEhBK3kxDvFDa2ou5atH0eUFjtOSszay2oBrCTVWK 7 | wZBsdUkL0HkW/wq9A8EUkQ8EownXnsxpI61CLNGLPpBZc+CRHhyWDD6BqwGvEHEv 8 | W546mh6k49//7zCiYfTK9/LCKBCFdDV6Sb7mNJ8HbNUj54uwC6iOgH25OCRDh4Bt 9 | zXoSrV9GLAm6AM25ZFo+ONOUBMtv7pavaR0CFMnAixl2NKV2wyLBYAYnJgdJFzGD 10 | 8mP6HwuR7e2g7PkcyC01EnX4iOIuuKHT/Xl9ynut4nHI7g6popotgashrQ5Jf8MS 11 | Kf98O12LzSMCAwEAAaOBhTCBgjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRf 12 | U9OxCBwCqYiUjWqY05sz3a6cmjBABgNVHSMEOTA3oSqkKDAmMSQwIgYDVQQDDBtz 13 | ZWNyZXQtb3BlcmF0b3Igc2VsZi1zaWduZWSCCQCfP9Hp7WlIJzAOBgNVHQ8BAf8E 14 | BAMCAYYwDQYJKoZIhvcNAQELBQADggEBAA8Flk1XOb1pH33Mbie5ronP2xw/xf6t 15 | Ox3PBEZ+5/jSPdIwoSaRp9JoP0L9Rg68jzcl5QMa4pOYWe+C1q8aZP0tjfq1eJfO 16 | UD5ik2DQgEuoF1ELgW1xoM38vkd8wgE711swDHK2zAsOudSzO4XZ4rQ6kaXXtoej 17 | 2kFhxDYcC+na90LdkJM0kAqrjxlFaP7WgUK+HA2iN00CFSOI9FVdppLtootbcb+y 18 | +WfXxM7gA9Exg4f2vKGVx7UxB/k4AbPvogBQZvK8VoAQocAhWrw7o2rqAesAw6JD 19 | WwQjM69TlEfbHYXtTfMbi01Wi5TtVhFCjyXK6KDsqSgU+9McExIy70k= 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/operations/graceful-shutdown.adoc: -------------------------------------------------------------------------------- 1 | = Graceful shutdown 2 | 3 | You can configure the graceful shutdown as described in xref:concepts:operations/graceful_shutdown.adoc[]. 4 | 5 | == Nodes 6 | 7 | As a default, NiFi nodes have `5 minutes` to shut down gracefully. 8 | 9 | The NiFi node process receives a `SIGTERM` signal when Kubernetes wants to terminate the Pod. 10 | It logs the received signal as shown in the log below and initiate a graceful shutdown. 11 | After the graceful shutdown timeout runs out, and the process still did not exit, Kubernetes issues a `SIGKILL` signal. 12 | 13 | [source,text] 14 | ---- 15 | nifi NiFi PID [43] shutdown started 16 | nifi NiFi PID [43] shutdown in progress... 17 | [...] 18 | nifi 2023-11-08 10:52:14,459 INFO [pool-1-thread-1] org.apache.nifi.NiFi Application Server shutdown completed 19 | nifi 2023-11-08 10:52:15,139 INFO [Thread-0] org.apache.nifi.NiFi Application Server shutdown started 20 | nifi 2023-11-08 10:52:15,139 INFO [Thread-0] org.apache.nifi.nar.NarAutoLoader NAR Auto-Loader stopped 21 | nifi 2023-11-08 10:52:15,139 INFO [Thread-0] o.a.n.f.r.CompositeExternalResourceProviderService External Resource Provider Service is stopped 22 | nifi 2023-11-08 10:52:15,139 INFO [Thread-0] org.apache.nifi.NiFi Application Server shutdown completed 23 | ---- 24 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | name: check-iceberg-tables 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: check-iceberg-tables 11 | image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-l'] }}-stackable0.0.0-dev" 12 | command: 13 | - bash 14 | - -euo 15 | - pipefail 16 | - -c 17 | - | 18 | for SCHEMA in iceberg.s3 iceberg.hdfs; do 19 | COUNT=$(cat << EOF | ./trino-cli --server https://trino-coordinator:8443 --insecure --user admin 20 | SELECT COUNT(*) FROM $SCHEMA.greetings WHERE hello = 'world from NiFi :)'; 21 | EOF 22 | ) 23 | 24 | COUNT="${COUNT%\"}" # Remove trailing quote if any 25 | COUNT="${COUNT#\"}" # Remove leading quote if any 26 | echo "Count is $COUNT" 27 | 28 | # Check if it's a number greater than 0 29 | if ! [[ "$COUNT" =~ ^[0-9]+$ ]] || [ "$COUNT" -le 0 ]; then 30 | echo "Invalid or zero count: $COUNT" 31 | exit 1 32 | fi 33 | 34 | echo "Count $COUNT was valid" 35 | done 36 | restartPolicy: OnFailure 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: secrets.stackable.tech/v1alpha1 2 | kind: SecretClass 3 | metadata: 4 | name: kerberos-$NAMESPACE 5 | spec: 6 | backend: 7 | kerberosKeytab: 8 | realmName: {{ test_scenario['values']['kerberos-realm'] }} 9 | kdc: krb5-kdc.$NAMESPACE.svc.cluster.local 10 | admin: 11 | mit: 12 | kadminServer: krb5-kdc.$NAMESPACE.svc.cluster.local 13 | adminKeytabSecret: 14 | namespace: $NAMESPACE 15 | name: secret-operator-keytab 16 | adminPrincipal: stackable-secret-operator 17 | --- 18 | apiVersion: v1 19 | kind: Secret 20 | metadata: 21 | name: secret-operator-keytab 22 | data: 23 | # To create keytab. When promted enter password asdf 24 | # cat | ktutil << 'EOF' 25 | # list 26 | # add_entry -password -p stackable-secret-operator@CLUSTER.LOCAL -k 1 -e aes256-cts-hmac-sha384-192 27 | # wkt /tmp/keytab 28 | # EOF 29 | {% if test_scenario['values']['kerberos-realm'] == 'CLUSTER.LOCAL' %} 30 | keytab: BQIAAABdAAEADUNMVVNURVIuTE9DQUwAGXN0YWNrYWJsZS1zZWNyZXQtb3BlcmF0b3IAAAABZAYWIgEAFAAgm8MCZ8B//XF1tH92GciD6/usWUNAmBTZnZQxLua2TkgAAAAB 31 | {% elif test_scenario['values']['kerberos-realm'] == 'PROD.MYCORP' %} 32 | keytab: BQIAAABbAAEAC1BST0QuTVlDT1JQABlzdGFja2FibGUtc2VjcmV0LW9wZXJhdG9yAAAAAWQZa0EBABQAIC/EnFNejq/K5lX6tX+B3/tkI13TCzkPB7d2ggCIEzE8AAAAAQ== 33 | {% endif %} 34 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/updating.adoc: -------------------------------------------------------------------------------- 1 | = Updating NiFi 2 | :description: Easily update or downgrade Apache NiFi on Kubernetes by changing the CRD version. 3 | 4 | Updating (or downgrading for that matter) the deployed version of NiFi is as simple as changing the version stated in the CRD. 5 | Continuing the example above, to change the deployed version from `1.27.0` to `2.0.0` you'd simply deploy the following CRD. 6 | 7 | [source,yaml] 8 | ---- 9 | apiVersion: nifi.stackable.tech/v1alpha1 10 | kind: NifiCluster 11 | metadata: 12 | name: simple-nifi 13 | spec: 14 | image: 15 | productVersion: 2.0.0 # <1> 16 | ---- 17 | 18 | <1> Change the NiFi version here 19 | 20 | [WARNING] 21 | ==== 22 | NiFi clusters cannot be upgraded or downgraded in a rolling fashion due to a limitation in NiFi prior to version 2. 23 | 24 | When upgrading between NiFi 1 versions or from NiFi 1 to NiFi 2, any change to the NiFi version in the CRD triggers a full cluster restart with brief downtime. 25 | However, the Stackable image version can be updated in a rolling manner, provided the NiFi version remains unchanged. 26 | 27 | For upgrades between NiFi 2 versions, e.g. from `2.0.0` to `2.4.0`, rolling upgrades are supported. 28 | ==== 29 | 30 | == NiFi 2.0.0 31 | 32 | Before you can upgrade to `2.0.0` you https://cwiki.apache.org/confluence/display/NIFI/Migration+Guidance[need to update] to at least version 1.27.x! 33 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/create_ldap_user.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # To check the existing users 4 | # ldapsearch -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin -b "ou=my users,dc=example,dc=org" 5 | 6 | # To check the new user 7 | # ldapsearch -H ldap://localhost:1389 -D "cn=integrationtest,ou=my users,dc=example,dc=org" -w 'bindPasswordWithSpecialCharacter\@<&>"'"'" -b "ou=my users,dc=example,dc=org" 8 | 9 | cat << 'EOF' | ldapadd -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin 10 | dn: ou=my users,dc=example,dc=org 11 | ou: my users 12 | objectclass: top 13 | objectclass: organizationalUnit 14 | EOF 15 | 16 | cat << 'EOF' | ldapadd -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin 17 | dn: cn=integrationtest,ou=my users,dc=example,dc=org 18 | objectClass: inetOrgPerson 19 | objectClass: posixAccount 20 | objectClass: shadowAccount 21 | cn: integrationtest 22 | uid: integrationtest 23 | givenName: Stackable 24 | sn: Integration-Test 25 | mail: integrationtest@stackable.de 26 | uidNumber: 16842 27 | gidNumber: 100 28 | homeDirectory: /home/integrationtest 29 | loginShell: /bin/bash 30 | userPassword: {crypt}x 31 | shadowLastChange: 0 32 | shadowMax: 0 33 | shadowWarning: 0 34 | EOF 35 | 36 | ldappasswd -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin -s 'bindPasswordWithSpecialCharacter\@<&>"'"'" "cn=integrationtest,ou=my users,dc=example,dc=org" 37 | -------------------------------------------------------------------------------- /.github/workflows/pr_pre-commit.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: pre-commit 3 | 4 | on: 5 | pull_request: 6 | merge_group: 7 | 8 | env: 9 | CARGO_TERM_COLOR: always 10 | NIX_PKG_MANAGER_VERSION: "2.30.0" 11 | RUST_TOOLCHAIN_VERSION: "nightly-2025-10-23" 12 | HADOLINT_VERSION: "v2.14.0" 13 | PYTHON_VERSION: "3.14" 14 | JINJA2_CLI_VERSION: "0.8.2" 15 | 16 | jobs: 17 | pre-commit: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Install host dependencies 21 | uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 22 | with: 23 | packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https 24 | version: ubuntu-latest 25 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 26 | with: 27 | persist-credentials: false 28 | submodules: recursive 29 | fetch-depth: 0 30 | - uses: stackabletech/actions/run-pre-commit@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 31 | with: 32 | python-version: ${{ env.PYTHON_VERSION }} 33 | rust: ${{ env.RUST_TOOLCHAIN_VERSION }} 34 | hadolint: ${{ env.HADOLINT_VERSION }} 35 | nix: ${{ env.NIX_PKG_MANAGER_VERSION }} 36 | nix-github-token: ${{ secrets.GITHUB_TOKEN }} 37 | jinja2-cli: ${{ env.JINJA2_CLI_VERSION }} 38 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pre-release-rust-deps.md: -------------------------------------------------------------------------------- 1 | ## Bump Rust Dependencies for Stackable Release YY.M.X 2 | 3 | 7 | 8 | 11 | 12 | Part of 13 | 14 | > [!NOTE] 15 | > During a Stackable release we need to update various Rust dependencies before 16 | > entering the final release period to ensure we run the latest versions of 17 | > crates. These bumps also include previously updated and released crates from 18 | > the `operator-rs` repository. 19 | 20 | ### Tasks 21 | 22 | - [ ] Bump Rust Dependencies, see below for more details. 23 | - [ ] Add changelog entry stating which important crates were bumped (including the version). 24 | 25 | > [!NOTE] 26 | > The bumping / updating of Rust dependencies is done in multiple steps: 27 | > 28 | > 1. Update the minimum Version in the root `Cargo.toml` manifest. 29 | > 2. Run the `cargo update` command, which also updates the `Cargo.lock` file. 30 | > 3. Lastly, run `make regenerate-nix` to update the `Cargo.nix` file. 31 | 32 | ### Bump Rust Dependencies 33 | 34 | - [ ] Bump `stackable-operator` and friends 35 | - [ ] Bump `product-config` 36 | - [ ] Bump all other dependencies 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/custom-components-git-sync/README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | This test case configures the NiFi cluster to fetch Python and Java 4 | processors via git-sync. A flow is deployed that include these 5 | processors. The test job triggers the flow and checks the result. 6 | 7 | # Custom processors 8 | 9 | ## Python processor "Greet" 10 | 11 | The Greet processor is implemented in Python and answers a request with 12 | the content "Hello!". 13 | 14 | Only NiFi 2 supports Python components. In the flow of NiFi 1, the 15 | Python processor is replaced with the internal ReplaceText processor. 16 | NiFi 1 ignores the Python configurations. 17 | 18 | ## Java processor "Shout" 19 | 20 | The Shout processor is implemented in Java and transforms the received 21 | content to uppercase. 22 | 23 | The processor is compiled with the dependencies for NiFi 2, but also 24 | works in NiFi 1 if compiled with JDK 11. 25 | 26 | # Flow 27 | 28 | ![NiFi canvas](./canvas.png) 29 | 30 | 1. The HandleHttpRequest processor listens for HTTP requests to 31 | `GET /greeting`. The request is forwarded to the Greet processor. 32 | 2. The Greet processor writes the content "Hello!" to the flow file that 33 | is then forwarded to the Shout processor. 34 | 3. The Shout processor transforms the content to uppercase. The flow 35 | file is forwarded to the HandleHttpResponse processor. 36 | 4. The HandleHttpResponse answers the HTTP request with this content. 37 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/33_hive.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: hive.stackable.tech/v1alpha1 3 | kind: HiveCluster 4 | metadata: 5 | name: hive 6 | spec: 7 | image: 8 | {% if test_scenario['values']['hive-l'].find(",") > 0 %} 9 | custom: "{{ test_scenario['values']['hive-l'].split(',')[1] }}" 10 | productVersion: "{{ test_scenario['values']['hive-l'].split(',')[0] }}" 11 | {% else %} 12 | productVersion: "{{ test_scenario['values']['hive-l'] }}" 13 | {% endif %} 14 | pullPolicy: IfNotPresent 15 | clusterConfig: 16 | database: 17 | connString: jdbc:postgresql://postgresql:5432/hive 18 | credentialsSecret: postgres-credentials 19 | dbType: postgres 20 | hdfs: 21 | configMap: hdfs 22 | s3: 23 | reference: minio 24 | {% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} 25 | authentication: 26 | kerberos: 27 | secretClass: kerberos-$NAMESPACE 28 | {% endif %} 29 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 30 | vectorAggregatorConfigMapName: vector-aggregator-discovery 31 | {% endif %} 32 | metastore: 33 | config: 34 | logging: 35 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 36 | roleGroups: 37 | default: 38 | replicas: 1 39 | --- 40 | apiVersion: v1 41 | kind: Secret 42 | metadata: 43 | name: postgres-credentials 44 | type: Opaque 45 | stringData: 46 | username: hive 47 | password: hive 48 | -------------------------------------------------------------------------------- /rust/operator-binary/src/operations/graceful_shutdown.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | 3 | use snafu::{ResultExt, Snafu}; 4 | use stackable_operator::builder::pod::PodBuilder; 5 | 6 | use crate::crd::NifiConfig; 7 | 8 | #[derive(Debug, Snafu)] 9 | pub enum Error { 10 | #[snafu(display("Failed to set terminationGracePeriod"))] 11 | SetTerminationGracePeriod { 12 | source: stackable_operator::builder::pod::Error, 13 | }, 14 | } 15 | 16 | pub fn graceful_shutdown_config_properties(config: &NifiConfig) -> BTreeMap { 17 | let mut graceful_shutdown_properties = BTreeMap::new(); 18 | if let Some(graceful_shutdown_timeout) = config.graceful_shutdown_timeout { 19 | graceful_shutdown_properties.insert( 20 | "graceful.shutdown.seconds".to_string(), 21 | graceful_shutdown_timeout.as_secs().to_string(), 22 | ); 23 | } 24 | graceful_shutdown_properties 25 | } 26 | 27 | pub fn add_graceful_shutdown_config( 28 | merged_config: &NifiConfig, 29 | pod_builder: &mut PodBuilder, 30 | ) -> Result<(), Error> { 31 | // This must be always set by the merge mechanism, as we provide a default value, 32 | // users can not disable graceful shutdown. 33 | if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout { 34 | pod_builder 35 | .termination_grace_period(&graceful_shutdown_timeout) 36 | .context(SetTerminationGracePeriodSnafu)?; 37 | } 38 | 39 | Ok(()) 40 | } 41 | -------------------------------------------------------------------------------- /docs/modules/nifi/partials/nav.adoc: -------------------------------------------------------------------------------- 1 | * xref:nifi:getting_started/index.adoc[] 2 | ** xref:nifi:getting_started/installation.adoc[] 3 | ** xref:nifi:getting_started/first_steps.adoc[] 4 | * xref:nifi:usage_guide/index.adoc[] 5 | ** xref:nifi:usage_guide/listenerclass.adoc[] 6 | ** xref:nifi:usage_guide/clustering.adoc[] 7 | ** xref:nifi:usage_guide/extra-volumes.adoc[] 8 | ** xref:nifi:usage_guide/security.adoc[] 9 | ** xref:nifi:usage_guide/resource-configuration.adoc[] 10 | ** xref:nifi:usage_guide/log-aggregation.adoc[] 11 | ** xref:nifi:usage_guide/monitoring.adoc[] 12 | ** xref:nifi:usage_guide/updating.adoc[] 13 | ** xref:nifi:usage_guide/overrides.adoc[] 14 | ** xref:nifi:usage_guide/writing-to-iceberg-tables.adoc[] 15 | ** xref:nifi:usage_guide/flow-versioning.adoc[] 16 | ** xref:nifi:usage_guide/exposing-processors/index.adoc[] 17 | *** xref:nifi:usage_guide/exposing-processors/http.adoc[] 18 | *** xref:nifi:usage_guide/exposing-processors/tcp.adoc[] 19 | ** xref:nifi:usage_guide/custom-components.adoc[] 20 | ** xref:nifi:usage_guide/operations/index.adoc[] 21 | *** xref:nifi:usage_guide/operations/cluster-operations.adoc[] 22 | *** xref:nifi:usage_guide/operations/pod-placement.adoc[] 23 | *** xref:nifi:usage_guide/operations/pod-disruptions.adoc[] 24 | *** xref:nifi:usage_guide/operations/graceful-shutdown.adoc[] 25 | * xref:nifi:troubleshooting/index.adoc[] 26 | * xref:nifi:reference/index.adoc[] 27 | ** xref:nifi:reference/crds.adoc[] 28 | *** {crd-docs}/nifi.stackable.tech/nificluster/v1alpha1/[NiFiCluster {external-link-icon}^] 29 | ** xref:nifi:reference/commandline-parameters.adoc[] 30 | ** xref:nifi:reference/environment-variables.adoc[] 31 | -------------------------------------------------------------------------------- /tests/templates/kuttl/logging/test_log_aggregation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import requests 3 | 4 | 5 | def check_sent_events(): 6 | response = requests.post( 7 | 'http://nifi-vector-aggregator:8686/graphql', 8 | json={ 9 | 'query': """ 10 | { 11 | transforms(first:100) { 12 | nodes { 13 | componentId 14 | metrics { 15 | sentEventsTotal { 16 | sentEventsTotal 17 | } 18 | } 19 | } 20 | } 21 | } 22 | """ 23 | } 24 | ) 25 | 26 | assert response.status_code == 200, \ 27 | 'Cannot access the API of the vector aggregator.' 28 | 29 | result = response.json() 30 | 31 | transforms = result['data']['transforms']['nodes'] 32 | for transform in transforms: 33 | sentEvents = transform['metrics']['sentEventsTotal'] 34 | componentId = transform['componentId'] 35 | 36 | if componentId == 'filteredInvalidEvents': 37 | assert sentEvents is None or \ 38 | sentEvents['sentEventsTotal'] == 0, \ 39 | 'Invalid log events were sent.' 40 | else: 41 | assert sentEvents is not None and \ 42 | sentEvents['sentEventsTotal'] > 0, \ 43 | f'No events were sent in "{componentId}".' 44 | 45 | 46 | if __name__ == '__main__': 47 | check_sent_events() 48 | print('Test successful!') 49 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["rust/operator-binary"] 3 | resolver = "2" 4 | 5 | [workspace.package] 6 | version = "0.0.0-dev" 7 | authors = ["Stackable GmbH "] 8 | license = "OSL-3.0" 9 | edition = "2021" 10 | repository = "https://github.com/stackabletech/nifi-operator" 11 | 12 | [workspace.dependencies] 13 | product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } 14 | stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.1" } 15 | 16 | anyhow = "1.0" 17 | built = { version = "0.8", features = ["chrono", "git2"] } 18 | clap = "4.5" 19 | const_format = "0.2" 20 | fnv = "1.0" 21 | futures = { version = "0.3", features = ["compat"] } 22 | indoc = "2.0" 23 | # We pin the kube version, as we use a patch for 2.0.1 below 24 | kube = "=2.0.1" 25 | pin-project = "1.1" 26 | rand = "0.9" 27 | rstest = "0.26" 28 | semver = "1.0" 29 | serde = { version = "1.0", features = ["derive"] } 30 | serde_json = "1.0" 31 | serde_yaml = "0.9" 32 | snafu = "0.8" 33 | strum = { version = "0.27", features = ["derive"] } 34 | tokio = { version = "1.40", features = ["full"] } 35 | tracing = "0.1" 36 | url = { version = "2.5.7" } 37 | xml-rs = "1.0" 38 | 39 | # [patch."https://github.com/stackabletech/operator-rs.git"] 40 | # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } 41 | # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } 42 | 43 | [patch.crates-io] 44 | kube = { git = "https://github.com/stackabletech/kube-rs", branch = "2.0.1-fix-schema-hoisting" } 45 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | let 2 | self = import ./. {}; 3 | inherit (self) sources pkgs meta; 4 | 5 | beku = pkgs.callPackage (sources."beku.py" + "/beku.nix") {}; 6 | cargoDependencySetOfCrate = crate: [ crate ] ++ pkgs.lib.concatMap cargoDependencySetOfCrate (crate.dependencies ++ crate.buildDependencies); 7 | cargoDependencySet = pkgs.lib.unique (pkgs.lib.flatten (pkgs.lib.mapAttrsToList (crateName: crate: cargoDependencySetOfCrate crate.build) self.cargo.workspaceMembers)); 8 | in pkgs.mkShell rec { 9 | name = meta.operator.name; 10 | 11 | packages = with pkgs; [ 12 | ## cargo et-al 13 | rustup # this breaks pkg-config if it is in the nativeBuildInputs 14 | cargo-udeps 15 | 16 | ## Extra dependencies for use in a pure env (nix-shell --pure) 17 | ## These are mosuly useful for maintainers of this shell.nix 18 | ## to ensure all the dependencies are caught. 19 | # cacert 20 | # vim nvim nano 21 | ]; 22 | 23 | # derivation runtime dependencies 24 | buildInputs = pkgs.lib.concatMap (crate: crate.buildInputs) cargoDependencySet; 25 | 26 | # build time dependencies 27 | nativeBuildInputs = pkgs.lib.concatMap (crate: crate.nativeBuildInputs) cargoDependencySet ++ (with pkgs; [ 28 | beku 29 | docker 30 | gettext # for the proper envsubst 31 | git 32 | jq 33 | kind 34 | kubectl 35 | kubernetes-helm 36 | kuttl 37 | nix # this is implied, but needed in the pure env 38 | # tilt already defined in default.nix 39 | which 40 | yq-go 41 | ]); 42 | 43 | LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; 44 | BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include"; 45 | } 46 | -------------------------------------------------------------------------------- /tests/templates/kuttl/cluster_operation/20-install-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: authentication.stackable.tech/v1alpha1 3 | kind: AuthenticationClass 4 | metadata: 5 | name: simple-nifi-users 6 | spec: 7 | provider: 8 | static: 9 | userCredentialsSecret: 10 | name: simple-nifi-admin-credentials 11 | --- 12 | apiVersion: v1 13 | kind: Secret 14 | metadata: 15 | name: simple-nifi-admin-credentials 16 | stringData: 17 | admin: supersecretpassword 18 | --- 19 | apiVersion: v1 20 | kind: Secret 21 | metadata: 22 | name: nifi-sensitive-property-key 23 | stringData: 24 | nifiSensitivePropsKey: mYsUp3rS3cr3tk3y 25 | --- 26 | apiVersion: nifi.stackable.tech/v1alpha1 27 | kind: NifiCluster 28 | metadata: 29 | name: test-nifi 30 | spec: 31 | image: 32 | {% if test_scenario['values']['nifi-latest'].find(",") > 0 %} 33 | custom: "{{ test_scenario['values']['nifi-latest'].split(',')[1] }}" 34 | productVersion: "{{ test_scenario['values']['nifi-latest'].split(',')[0] }}" 35 | {% else %} 36 | custom: null 37 | productVersion: "{{ test_scenario['values']['nifi-latest'] }}" 38 | {% endif %} 39 | pullPolicy: IfNotPresent 40 | clusterConfig: 41 | authentication: 42 | - authenticationClass: simple-nifi-users 43 | sensitiveProperties: 44 | keySecret: nifi-sensitive-property-key 45 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 46 | vectorAggregatorConfigMapName: vector-aggregator-discovery 47 | {% endif %} 48 | zookeeperConfigMapName: test-nifi-znode 49 | nodes: 50 | config: 51 | gracefulShutdownTimeout: 1m 52 | logging: 53 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 54 | roleGroups: 55 | default: 56 | replicas: 2 57 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/02-install-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: authentication.stackable.tech/v1alpha1 3 | kind: AuthenticationClass 4 | metadata: 5 | name: simple-nifi-users 6 | spec: 7 | provider: 8 | static: 9 | userCredentialsSecret: 10 | name: simple-nifi-admin-credentials 11 | --- 12 | apiVersion: v1 13 | kind: Secret 14 | metadata: 15 | name: simple-nifi-admin-credentials 16 | stringData: 17 | admin: supersecretpassword 18 | --- 19 | apiVersion: v1 20 | kind: Secret 21 | metadata: 22 | name: nifi-sensitive-property-key 23 | stringData: 24 | nifiSensitivePropsKey: mYsUp3rS3cr3tk3y 25 | --- 26 | apiVersion: nifi.stackable.tech/v1alpha1 27 | kind: NifiCluster 28 | metadata: 29 | name: test-nifi 30 | spec: 31 | image: 32 | {% if test_scenario['values']['nifi_old'].find(",") > 0 %} 33 | custom: "{{ test_scenario['values']['nifi_old'].split(',')[1] }}" 34 | productVersion: "{{ test_scenario['values']['nifi_old'].split(',')[0] }}" 35 | {% else %} 36 | custom: null 37 | productVersion: "{{ test_scenario['values']['nifi_old'] }}" 38 | {% endif %} 39 | pullPolicy: IfNotPresent 40 | clusterConfig: 41 | authentication: 42 | - authenticationClass: simple-nifi-users 43 | sensitiveProperties: 44 | keySecret: nifi-sensitive-property-key 45 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 46 | vectorAggregatorConfigMapName: vector-aggregator-discovery 47 | {% endif %} 48 | zookeeperConfigMapName: test-nifi-znode 49 | nodes: 50 | config: 51 | gracefulShutdownTimeout: 1m 52 | logging: 53 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 54 | roleGroups: 55 | default: 56 | config: {} 57 | replicas: 3 58 | -------------------------------------------------------------------------------- /tests/templates/kuttl/iceberg/README.md: -------------------------------------------------------------------------------- 1 | The file `60_nifi-flow.json` was exported from the NiFi UI. 2 | 3 | *However*, we need to update some stuff, such as adding S3 credentials and templating the namespace of MinIO. 4 | 5 | TIP: I used `JSON: Sort Document` in VScode to somewhat have consistent formatting, which makes reading and diffs easier. 6 | 7 | Notable the following diff has been made (may not be up to date!): 8 | 9 | ```diff 10 | diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow.json b/tests/templates/kuttl/iceberg/60_nifi-flow.json 11 | index 09783fa..23c679f 100644 12 | --- a/tests/templates/kuttl/iceberg/60_nifi-flow.json 13 | +++ b/tests/templates/kuttl/iceberg/60_nifi-flow.json 14 | @@ -160,7 +160,9 @@ 15 | "custom-signer-module-location": null, 16 | "default-credentials": "false", 17 | "profile-name": null, 18 | - "Session Time": "3600" 19 | + "Session Time": "3600", 20 | + "Access Key": "admin", 21 | + "Secret Key": "adminadmin" 22 | }, 23 | "propertyDescriptors": { 24 | "Access Key": { 25 | @@ -483,7 +485,7 @@ 26 | "properties": { 27 | "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", 28 | "hive-metastore-uri": "thrift://hive-metastore:9083", 29 | - "s3-endpoint": "https://minio.kuttl-test-patient-tarpon.svc.cluster.local:9000", 30 | + "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", 31 | "s3-path-style-access": "true", 32 | "warehouse-location": "s3a://demo/lakehouse" 33 | }, 34 | ``` 35 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | *Please add a description here. This will become the commit message of the merge request later.* 4 | 5 | ## Definition of Done Checklist 6 | 7 | - Not all of these items are applicable to all PRs, the author should update this template to only leave the boxes in that are relevant 8 | - Please make sure all these things are done and tick the boxes 9 | 10 | ### Author 11 | 12 | - [ ] Changes are OpenShift compatible 13 | - [ ] CRD changes approved 14 | - [ ] CRD documentation for all fields, following the [style guide](https://docs.stackable.tech/home/nightly/contributor/docs/style-guide). 15 | - [ ] Helm chart can be installed and deployed operator works 16 | - [ ] Integration tests passed (for non trivial changes) 17 | - [ ] Changes need to be "offline" compatible 18 | - [ ] Links to generated (nightly) docs added 19 | - [ ] Release note snippet added 20 | 21 | ### Reviewer 22 | 23 | - [ ] Code contains useful comments 24 | - [ ] Code contains useful logging statements 25 | - [ ] (Integration-)Test cases added 26 | - [ ] Documentation added or updated. Follows the [style guide](https://docs.stackable.tech/home/nightly/contributor/docs/style-guide). 27 | - [ ] Changelog updated 28 | - [ ] Cargo.toml only contains references to git tags (not specific commits or branches) 29 | 30 | ### Acceptance 31 | 32 | - [ ] Feature Tracker has been updated 33 | - [ ] Proper release label has been added 34 | - [ ] Links to generated (nightly) docs added 35 | - [ ] Release note snippet added 36 | - [ ] Add `type/deprecation` label & add to the [deprecation schedule](https://github.com/orgs/stackabletech/projects/44/views/1) 37 | - [ ] Add `type/experimental` label & add to the [experimental features tracker](https://github.com/orgs/stackabletech/projects/47) 38 | -------------------------------------------------------------------------------- /examples/simple-cluster/simple-nifi-cluster.yaml: -------------------------------------------------------------------------------- 1 | # Deploy a NiFi cluster, a user secret and a ZooKeeper cluster and a respective 2 | # ZNode `simple-nifi-znode` which will be referenced 3 | --- 4 | apiVersion: zookeeper.stackable.tech/v1alpha1 5 | kind: ZookeeperCluster 6 | metadata: 7 | name: simple-zk 8 | spec: 9 | image: 10 | productVersion: 3.9.4 11 | servers: 12 | roleGroups: 13 | default: 14 | replicas: 1 15 | --- 16 | apiVersion: zookeeper.stackable.tech/v1alpha1 17 | kind: ZookeeperZnode 18 | metadata: 19 | name: simple-nifi-znode 20 | spec: 21 | clusterRef: 22 | name: simple-zk 23 | --- 24 | apiVersion: authentication.stackable.tech/v1alpha1 25 | kind: AuthenticationClass 26 | metadata: 27 | name: simple-nifi-admin-user 28 | spec: 29 | provider: 30 | static: 31 | userCredentialsSecret: 32 | name: simple-nifi-admin-user-secret 33 | --- 34 | apiVersion: v1 35 | kind: Secret 36 | metadata: 37 | name: simple-nifi-admin-user-secret 38 | stringData: 39 | # The admin user is required for a working single user NiFi deployment. The password can be changed as desired. 40 | admin: admin 41 | # Will not be read and used by NiFi (e.g. in case of reusing a Secret) 42 | bob: bob 43 | --- 44 | apiVersion: nifi.stackable.tech/v1alpha1 45 | kind: NifiCluster 46 | metadata: 47 | name: simple-nifi 48 | spec: 49 | image: 50 | productVersion: 2.6.0 51 | clusterConfig: 52 | authentication: 53 | - authenticationClass: simple-nifi-admin-user 54 | sensitiveProperties: 55 | keySecret: nifi-sensitive-property-key 56 | autoGenerate: true 57 | zookeeperConfigMapName: simple-nifi-znode 58 | nodes: 59 | roleConfig: 60 | listenerClass: external-unstable 61 | roleGroups: 62 | default: 63 | replicas: 1 64 | -------------------------------------------------------------------------------- /tests/templates/kuttl/orphaned_resources/02-install-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: authentication.stackable.tech/v1alpha1 3 | kind: AuthenticationClass 4 | metadata: 5 | name: simple-nifi-users 6 | spec: 7 | provider: 8 | static: 9 | userCredentialsSecret: 10 | name: simple-nifi-admin-credentials 11 | --- 12 | apiVersion: v1 13 | kind: Secret 14 | metadata: 15 | name: simple-nifi-admin-credentials 16 | stringData: 17 | admin: supersecretpassword 18 | --- 19 | apiVersion: v1 20 | kind: Secret 21 | metadata: 22 | name: nifi-sensitive-property-key 23 | stringData: 24 | nifiSensitivePropsKey: mYsUp3rS3cr3tk3y 25 | --- 26 | apiVersion: nifi.stackable.tech/v1alpha1 27 | kind: NifiCluster 28 | metadata: 29 | name: test-nifi 30 | spec: 31 | image: 32 | {% if test_scenario['values']['nifi'].find(",") > 0 %} 33 | custom: "{{ test_scenario['values']['nifi'].split(',')[1] }}" 34 | productVersion: "{{ test_scenario['values']['nifi'].split(',')[0] }}" 35 | {% else %} 36 | custom: null 37 | productVersion: "{{ test_scenario['values']['nifi'] }}" 38 | {% endif %} 39 | pullPolicy: IfNotPresent 40 | clusterConfig: 41 | authentication: 42 | - authenticationClass: simple-nifi-users 43 | sensitiveProperties: 44 | keySecret: nifi-sensitive-property-key 45 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 46 | vectorAggregatorConfigMapName: vector-aggregator-discovery 47 | {% endif %} 48 | zookeeperConfigMapName: test-nifi-znode 49 | nodes: 50 | config: 51 | gracefulShutdownTimeout: 1m 52 | logging: 53 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 54 | roleGroups: 55 | default: 56 | config: {} 57 | replicas: 2 58 | throwaway: 59 | config: {} 60 | replicas: 1 61 | -------------------------------------------------------------------------------- /nix/sources.json: -------------------------------------------------------------------------------- 1 | { 2 | "beku.py": { 3 | "branch": "0.0.10", 4 | "description": "Test suite expander for Stackable Kuttl tests.", 5 | "homepage": null, 6 | "owner": "stackabletech", 7 | "repo": "beku.py", 8 | "rev": "fc75202a38529a4ac6776dd8a5dfee278d927f58", 9 | "sha256": "152yary0p11h87yabv74jnwkghsal7lx16az0qlzrzdrs6n5v8id", 10 | "type": "tarball", 11 | "url": "https://github.com/stackabletech/beku.py/archive/fc75202a38529a4ac6776dd8a5dfee278d927f58.tar.gz", 12 | "url_template": "https://github.com///archive/.tar.gz" 13 | }, 14 | "crate2nix": { 15 | "branch": "master", 16 | "description": "nix build file generator for rust crates", 17 | "homepage": "", 18 | "owner": "kolloch", 19 | "repo": "crate2nix", 20 | "rev": "be31feae9a82c225c0fd1bdf978565dc452a483a", 21 | "sha256": "14d0ymlrwk7dynv35qcw4xn0dylfpwjmf6f8znflbk2l6fk23l12", 22 | "type": "tarball", 23 | "url": "https://github.com/kolloch/crate2nix/archive/be31feae9a82c225c0fd1bdf978565dc452a483a.tar.gz", 24 | "url_template": "https://github.com///archive/.tar.gz" 25 | }, 26 | "nixpkgs": { 27 | "branch": "nixpkgs-unstable", 28 | "description": "Nix Packages collection", 29 | "homepage": "", 30 | "owner": "NixOS", 31 | "repo": "nixpkgs", 32 | "rev": "a7fc11be66bdfb5cdde611ee5ce381c183da8386", 33 | "sha256": "0h3gvjbrlkvxhbxpy01n603ixv0pjy19n9kf73rdkchdvqcn70j2", 34 | "type": "tarball", 35 | "url": "https://github.com/NixOS/nixpkgs/archive/a7fc11be66bdfb5cdde611ee5ce381c183da8386.tar.gz", 36 | "url_template": "https://github.com///archive/.tar.gz" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tests/templates/kuttl/external-access/30_nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: authentication.stackable.tech/v1alpha1 3 | kind: AuthenticationClass 4 | metadata: 5 | name: simple-nifi-users 6 | spec: 7 | provider: 8 | static: 9 | userCredentialsSecret: 10 | name: simple-nifi-admin-credentials 11 | --- 12 | apiVersion: v1 13 | kind: Secret 14 | metadata: 15 | name: simple-nifi-admin-credentials 16 | stringData: 17 | admin: > 18 | passwordWithSpecialCharacter\@<&>"' 19 | --- 20 | apiVersion: v1 21 | kind: Secret 22 | metadata: 23 | name: nifi-sensitive-property-key 24 | stringData: 25 | nifiSensitivePropsKey: mYsUp3rS3cr3tk3y 26 | --- 27 | apiVersion: nifi.stackable.tech/v1alpha1 28 | kind: NifiCluster 29 | metadata: 30 | name: test-nifi 31 | spec: 32 | image: 33 | {% if test_scenario['values']['nifi'].find(",") > 0 %} 34 | custom: "{{ test_scenario['values']['nifi'].split(',')[1] }}" 35 | productVersion: "{{ test_scenario['values']['nifi'].split(',')[0] }}" 36 | {% else %} 37 | custom: null 38 | productVersion: "{{ test_scenario['values']['nifi'] }}" 39 | {% endif %} 40 | pullPolicy: IfNotPresent 41 | clusterConfig: 42 | zookeeperConfigMapName: test-nifi-znode 43 | authentication: 44 | - authenticationClass: simple-nifi-users 45 | hostHeaderCheck: 46 | allowAll: false 47 | sensitiveProperties: 48 | keySecret: nifi-sensitive-property-key 49 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 50 | vectorAggregatorConfigMapName: vector-aggregator-discovery 51 | {% endif %} 52 | nodes: 53 | config: 54 | logging: 55 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 56 | roleConfig: 57 | listenerClass: test-external-unstable-$NAMESPACE 58 | roleGroups: 59 | default: 60 | replicas: 2 61 | -------------------------------------------------------------------------------- /tests/templates/kuttl/ldap/12-install-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: nifi-sensitive-property-key 6 | stringData: 7 | nifiSensitivePropsKey: mYsUp3rS3cr3tk3y 8 | --- 9 | apiVersion: v1 10 | kind: Secret 11 | metadata: 12 | name: nifi-with-ldap-bind-secret 13 | labels: 14 | secrets.stackable.tech/class: nifi-with-ldap-bind 15 | stringData: 16 | user: cn=integrationtest,ou=my users,dc=example,dc=org 17 | password: > 18 | bindPasswordWithSpecialCharacter\@<&>"' 19 | --- 20 | apiVersion: nifi.stackable.tech/v1alpha1 21 | kind: NifiCluster 22 | metadata: 23 | name: test-nifi 24 | spec: 25 | image: 26 | {% if test_scenario['values']['nifi'].find(",") > 0 %} 27 | custom: "{{ test_scenario['values']['nifi'].split(',')[1] }}" 28 | productVersion: "{{ test_scenario['values']['nifi'].split(',')[0] }}" 29 | {% else %} 30 | custom: null 31 | productVersion: "{{ test_scenario['values']['nifi'] }}" 32 | {% endif %} 33 | pullPolicy: IfNotPresent 34 | clusterConfig: 35 | authentication: 36 | {% if test_scenario['values']['ldap-use-tls'] == 'false' %} 37 | - authenticationClass: ldap-without-tls 38 | {% else %} 39 | - authenticationClass: ldap-with-tls 40 | {% endif %} 41 | sensitiveProperties: 42 | keySecret: nifi-sensitive-property-key 43 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 44 | vectorAggregatorConfigMapName: vector-aggregator-discovery 45 | {% endif %} 46 | zookeeperConfigMapName: nifi-with-ldap-znode 47 | nodes: 48 | config: 49 | gracefulShutdownTimeout: 1m 50 | logging: 51 | enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} 52 | roleConfig: 53 | listenerClass: external-unstable 54 | roleGroups: 55 | default: 56 | config: {} 57 | replicas: 2 58 | -------------------------------------------------------------------------------- /tests/templates/kuttl/resources/02-assert.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestAssert 4 | # It sometimes takes a long time to provision 15 PersistentVolumes. 5 | timeout: 1800 6 | --- 7 | apiVersion: apps/v1 8 | kind: StatefulSet 9 | metadata: 10 | name: test-nifi-node-resources-from-role 11 | spec: 12 | template: 13 | spec: 14 | containers: 15 | - name: nifi 16 | resources: 17 | requests: 18 | cpu: 500m 19 | memory: 2Gi 20 | limits: 21 | cpu: "1" 22 | memory: 2Gi 23 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 24 | - name: vector 25 | {% endif %} 26 | status: 27 | readyReplicas: 1 28 | replicas: 1 29 | --- 30 | apiVersion: apps/v1 31 | kind: StatefulSet 32 | metadata: 33 | name: test-nifi-node-resources-from-role-group 34 | spec: 35 | template: 36 | spec: 37 | containers: 38 | - name: nifi 39 | resources: 40 | requests: 41 | cpu: 600m 42 | memory: 3Gi 43 | limits: 44 | cpu: 1100m 45 | memory: 3Gi 46 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 47 | - name: vector 48 | {% endif %} 49 | status: 50 | readyReplicas: 1 51 | replicas: 1 52 | --- 53 | apiVersion: apps/v1 54 | kind: StatefulSet 55 | metadata: 56 | name: test-nifi-node-resources-from-pod-overrides 57 | spec: 58 | template: 59 | spec: 60 | containers: 61 | - name: nifi 62 | resources: 63 | requests: 64 | cpu: 700m 65 | memory: 2Gi 66 | limits: 67 | cpu: 1200m 68 | memory: 2Gi 69 | {% if lookup('env', 'VECTOR_AGGREGATOR') %} 70 | - name: vector 71 | {% endif %} 72 | status: 73 | readyReplicas: 1 74 | replicas: 1 75 | -------------------------------------------------------------------------------- /.readme/static/borrowed/Icon_Stackable.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /deploy/helm/nifi-operator/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for nifi-operator. 2 | --- 3 | image: 4 | repository: oci.stackable.tech/sdp/nifi-operator 5 | pullPolicy: IfNotPresent 6 | pullSecrets: [] 7 | 8 | nameOverride: "" 9 | fullnameOverride: "" 10 | 11 | serviceAccount: 12 | # Specifies whether a service account should be created 13 | create: true 14 | # Annotations to add to the service account 15 | annotations: {} 16 | # The name of the service account to use. 17 | # If not set and create is true, a name is generated using the fullname template 18 | name: "" 19 | 20 | podAnnotations: {} 21 | 22 | # Provide additional labels which get attached to all deployed resources 23 | labels: 24 | stackable.tech/vendor: Stackable 25 | 26 | podSecurityContext: {} 27 | # fsGroup: 2000 28 | 29 | securityContext: {} 30 | # capabilities: 31 | # drop: 32 | # - ALL 33 | # readOnlyRootFilesystem: true 34 | # runAsNonRoot: true 35 | # runAsUser: 1000 36 | 37 | resources: 38 | limits: 39 | cpu: 100m 40 | memory: 128Mi 41 | requests: 42 | cpu: 100m 43 | memory: 128Mi 44 | 45 | nodeSelector: {} 46 | 47 | tolerations: [] 48 | 49 | affinity: {} 50 | 51 | # When running on a non-default Kubernetes cluster domain, the cluster domain can be configured here. 52 | # See the https://docs.stackable.tech/home/stable/guides/kubernetes-cluster-domain guide for details. 53 | # kubernetesClusterDomain: my-cluster.local# kubernetesClusterDomain: my-cluster.local 54 | 55 | # See all available options and detailed explanations about the concept here: 56 | # https://docs.stackable.tech/home/stable/concepts/telemetry/ 57 | telemetry: 58 | consoleLog: 59 | enabled: true 60 | fileLog: 61 | enabled: false 62 | rotationPeriod: hourly 63 | maxFiles: 6 64 | otelLogExporter: 65 | enabled: false 66 | otelTraceExporter: 67 | enabled: false 68 | -------------------------------------------------------------------------------- /rust/operator-binary/src/operations/pdb.rs: -------------------------------------------------------------------------------- 1 | use snafu::{ResultExt, Snafu}; 2 | use stackable_operator::{ 3 | builder::pdb::PodDisruptionBudgetBuilder, client::Client, cluster_resources::ClusterResources, 4 | commons::pdb::PdbConfig, kube::ResourceExt, 5 | }; 6 | 7 | use crate::{ 8 | OPERATOR_NAME, 9 | controller::NIFI_CONTROLLER_NAME, 10 | crd::{APP_NAME, NifiRole, v1alpha1}, 11 | }; 12 | 13 | #[derive(Snafu, Debug)] 14 | pub enum Error { 15 | #[snafu(display("Cannot create PodDisruptionBudget for role [{role}]"))] 16 | CreatePdb { 17 | source: stackable_operator::builder::pdb::Error, 18 | role: String, 19 | }, 20 | #[snafu(display("Cannot apply PodDisruptionBudget [{name}]"))] 21 | ApplyPdb { 22 | source: stackable_operator::cluster_resources::Error, 23 | name: String, 24 | }, 25 | } 26 | 27 | pub async fn add_pdbs( 28 | pdb: &PdbConfig, 29 | nifi: &v1alpha1::NifiCluster, 30 | role: &NifiRole, 31 | client: &Client, 32 | cluster_resources: &mut ClusterResources, 33 | ) -> Result<(), Error> { 34 | if !pdb.enabled { 35 | return Ok(()); 36 | } 37 | let max_unavailable = pdb.max_unavailable.unwrap_or(match role { 38 | NifiRole::Node => max_unavailable_nodes(), 39 | }); 40 | let pdb = PodDisruptionBudgetBuilder::new_with_role( 41 | nifi, 42 | APP_NAME, 43 | &role.to_string(), 44 | OPERATOR_NAME, 45 | NIFI_CONTROLLER_NAME, 46 | ) 47 | .with_context(|_| CreatePdbSnafu { 48 | role: role.to_string(), 49 | })? 50 | .with_max_unavailable(max_unavailable) 51 | .build(); 52 | let pdb_name = pdb.name_any(); 53 | cluster_resources 54 | .add(client, pdb) 55 | .await 56 | .with_context(|_| ApplyPdbSnafu { name: pdb_name })?; 57 | 58 | Ok(()) 59 | } 60 | 61 | fn max_unavailable_nodes() -> u16 { 62 | 1 63 | } 64 | -------------------------------------------------------------------------------- /rust/operator-binary/src/security/mod.rs: -------------------------------------------------------------------------------- 1 | use snafu::{ResultExt, Snafu}; 2 | use stackable_operator::{ 3 | builder::pod::volume::SecretFormat, client::Client, k8s_openapi::api::core::v1::Volume, 4 | shared::time::Duration, 5 | }; 6 | 7 | use crate::crd::v1alpha1; 8 | 9 | pub mod authentication; 10 | pub mod authorization; 11 | pub mod oidc; 12 | pub mod sensitive_key; 13 | pub mod tls; 14 | 15 | type Result = std::result::Result; 16 | 17 | #[derive(Snafu, Debug)] 18 | pub enum Error { 19 | #[snafu(display("tls failure"))] 20 | Tls { source: tls::Error }, 21 | 22 | #[snafu(display("sensistive key failure"))] 23 | SensitiveKey { source: sensitive_key::Error }, 24 | 25 | #[snafu(display("failed to ensure OIDC admin password exists"))] 26 | OidcAdminPassword { source: oidc::Error }, 27 | } 28 | 29 | pub async fn check_or_generate_sensitive_key( 30 | client: &Client, 31 | nifi: &v1alpha1::NifiCluster, 32 | ) -> Result { 33 | sensitive_key::check_or_generate_sensitive_key(client, nifi) 34 | .await 35 | .context(SensitiveKeySnafu) 36 | } 37 | 38 | pub async fn check_or_generate_oidc_admin_password( 39 | client: &Client, 40 | nifi: &v1alpha1::NifiCluster, 41 | ) -> Result { 42 | oidc::check_or_generate_oidc_admin_password(client, nifi) 43 | .await 44 | .context(OidcAdminPasswordSnafu) 45 | } 46 | 47 | pub fn build_tls_volume( 48 | nifi: &v1alpha1::NifiCluster, 49 | volume_name: &str, 50 | service_scopes: impl IntoIterator>, 51 | secret_format: SecretFormat, 52 | requested_secret_lifetime: &Duration, 53 | listener_scope: Option<&str>, 54 | ) -> Result { 55 | tls::build_tls_volume( 56 | nifi, 57 | volume_name, 58 | service_scopes, 59 | secret_format, 60 | requested_secret_lifetime, 61 | listener_scope, 62 | ) 63 | .context(TlsSnafu) 64 | } 65 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/02-bug_report.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "🐛 Bug Report" 3 | description: "If something isn't working as expected 🤔." 4 | labels: ["type/bug"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: Thanks for taking the time to file a bug report! Please fill out this form as completely as possible. 9 | 10 | - type: input 11 | attributes: 12 | label: Affected Stackable version 13 | description: Which version of the Stackable Operator do you see this bug in? 14 | 15 | # 16 | - type: input 17 | attributes: 18 | label: Affected Apache NiFi version 19 | description: Which version of Apache NiFi do you see this bug in? 20 | # 21 | 22 | - type: textarea 23 | attributes: 24 | label: Current and expected behavior 25 | description: A clear and concise description of what the operator is doing and what you would expect. 26 | validations: 27 | required: true 28 | 29 | - type: textarea 30 | attributes: 31 | label: Possible solution 32 | description: "If you have suggestions on a fix for the bug." 33 | 34 | - type: textarea 35 | attributes: 36 | label: Additional context 37 | description: "Add any other context about the problem here. Or a screenshot if applicable." 38 | 39 | - type: textarea 40 | attributes: 41 | label: Environment 42 | description: | 43 | What type of kubernetes cluster you are running against (k3s/eks/aks/gke/other) and any other information about your environment? 44 | placeholder: | 45 | Examples: 46 | Output of `kubectl version --short` 47 | 48 | - type: dropdown 49 | attributes: 50 | label: Would you like to work on fixing this bug? 51 | description: | 52 | **NOTE**: Let us know if you would like to submit a PR for this. We are more than happy to help you through the process. 53 | options: 54 | - "yes" 55 | - "no" 56 | - "maybe" 57 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/usage_guide/resource-configuration.adoc: -------------------------------------------------------------------------------- 1 | = Resource configuration 2 | :description: Configure NiFi storage and resource requests for optimal performance. Set PVC sizes and resource requests for CPU, memory, and persistent storage. 3 | 4 | == Volume storage 5 | 6 | By default, a NiFi Stacklet creates five different persistent volume claims for flow files, provenance, database, content and state directories. 7 | You can find the default sizes of the PVCs in the {crd-docs}/nifi.stackable.tech/NifiCluster/v1alpha1/#spec-nodes-config-resources-storage[NifiCluster reference docs {external-link-icon}^], it is recommended that you configure these volume requests according to your needs. 8 | 9 | Storage requests can be configured at role or group level, for one or more of the persistent volumes as follows: 10 | 11 | [source,yaml] 12 | ---- 13 | nodes: 14 | roleGroups: 15 | default: 16 | config: 17 | resources: 18 | storage: 19 | flowfileRepo: 20 | capacity: 12Gi 21 | provenanceRepo: 22 | capacity: 12Gi 23 | databaseRepo: 24 | capacity: 12Gi 25 | contentRepo: 26 | capacity: 12Gi 27 | stateRepo: 28 | capacity: 12Gi 29 | ---- 30 | 31 | In the above example, all nodes in the default group request `12Gi` of storage the various directories. 32 | 33 | == Resource requests 34 | 35 | include::home:concepts:stackable_resource_requests.adoc[] 36 | 37 | A minimal HA setup consisting of 2 NiFi instances has the following https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/[resource requirements]: 38 | 39 | * `1500m` CPU request 40 | * `5000m` CPU limit 41 | * `8448Mi` memory request and limit 42 | * `18432Mi` persistent storage 43 | 44 | You can find the default resource configuration values in the {crd-docs}/nifi.stackable.tech/NifiCluster/v1alpha1/#spec-nodes-config-resources[NifiCluster reference docs {external-link-icon}^]. 45 | -------------------------------------------------------------------------------- /deploy/helm/nifi-operator/templates/_telemetry.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Create a list of telemetry related env vars. 3 | */}} 4 | {{- define "telemetry.envVars" -}} 5 | {{- with .Values.telemetry }} 6 | {{- if not .consoleLog.enabled }} 7 | - name: CONSOLE_LOG_DISABLED 8 | value: "true" 9 | {{- end }} 10 | {{- if and .consoleLog.enabled .consoleLog.level }} 11 | - name: CONSOLE_LOG_LEVEL 12 | value: {{ .consoleLog.level }} 13 | {{ end }} 14 | {{- if and .consoleLog.enabled .consoleLog.format }} 15 | - name: CONSOLE_LOG_FORMAT 16 | value: {{ .consoleLog.format }} 17 | {{ end }} 18 | {{- if .fileLog.enabled }} 19 | - name: FILE_LOG_DIRECTORY 20 | value: /stackable/logs/{{ include "operator.appname" $ }} 21 | {{- end }} 22 | {{- if and .fileLog.enabled .fileLog.level }} 23 | - name: FILE_LOG_LEVEL 24 | value: {{ .fileLog.level }} 25 | {{- end }} 26 | {{- if and .fileLog.enabled .fileLog.rotationPeriod }} 27 | - name: FILE_LOG_ROTATION_PERIOD 28 | value: {{ .fileLog.rotationPeriod }} 29 | {{- end }} 30 | {{- if and .fileLog.enabled .fileLog.maxFiles }} 31 | - name: FILE_LOG_MAX_FILES 32 | value: {{ quote .fileLog.maxFiles }} 33 | {{- end }} 34 | {{- if .otelLogExporter.enabled }} 35 | - name: OTEL_LOG_EXPORTER_ENABLED 36 | value: "true" 37 | {{- end }} 38 | {{- if and .otelLogExporter.enabled .otelLogExporter.level }} 39 | - name: OTEL_LOG_EXPORTER_LEVEL 40 | value: {{ .otelLogExporter.level }} 41 | {{- end }} 42 | {{- if and .otelLogExporter.enabled .otelLogExporter.endpoint }} 43 | - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT 44 | value: {{ .otelLogExporter.endpoint }} 45 | {{- end }} 46 | {{- if .otelTraceExporter.enabled }} 47 | - name: OTEL_TRACE_EXPORTER_ENABLED 48 | value: "true" 49 | {{- end }} 50 | {{- if and .otelTraceExporter.enabled .otelTraceExporter.level }} 51 | - name: OTEL_TRACE_EXPORTER_LEVEL 52 | value: {{ .otelTraceExporter.level }} 53 | {{- end }} 54 | {{- if and .otelTraceExporter.enabled .otelTraceExporter.endpoint }} 55 | - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT 56 | value: {{ .otelTraceExporter.endpoint }} 57 | {{- end }} 58 | {{- end }} 59 | {{- end }} 60 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/20-install-opa.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kuttl.dev/v1beta1 3 | kind: TestStep 4 | commands: 5 | - script: | 6 | kubectl apply -n $NAMESPACE -f - < 0 }} 56 | containers: 57 | opa: 58 | loggers: 59 | decision: 60 | level: INFO 61 | roleGroups: 62 | default: {} 63 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v1/test_nifi_metrics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import requests 4 | import time 5 | from requests.exceptions import ConnectionError 6 | 7 | if __name__ == "__main__": 8 | # Construct an argument parser 9 | all_args = argparse.ArgumentParser() 10 | # Add arguments to the parser 11 | all_args.add_argument( 12 | "-m", 13 | "--metric", 14 | required=False, 15 | default="nifi_amount_bytes_read", 16 | help="The name of a certain metric to check", 17 | ) 18 | all_args.add_argument( 19 | "-n", "--namespace", required=True, help="The namespace the test is running in" 20 | ) 21 | all_args.add_argument( 22 | "-p", 23 | "--port", 24 | required=False, 25 | default="8081", 26 | help="The port where metrics are exposed", 27 | ) 28 | all_args.add_argument( 29 | "-t", 30 | "--timeout", 31 | required=False, 32 | default="120", 33 | help="The timeout in seconds to wait for the metrics port to be opened", 34 | ) 35 | 36 | args = vars(all_args.parse_args()) 37 | metric_name = args["metric"] 38 | namespace = args["namespace"] 39 | port = args["port"] 40 | timeout = int(args["timeout"]) 41 | 42 | url = f"http://nifi-node-default-metrics.{namespace}.svc.cluster.local:{port}/metrics/" 43 | 44 | # wait for 'timeout' seconds 45 | t_end = time.time() + timeout 46 | while time.time() < t_end: 47 | try: 48 | response = requests.get(url) 49 | response.raise_for_status() 50 | if metric_name in response.text: 51 | print("Test metrics succeeded!") 52 | exit(0) 53 | else: 54 | print( 55 | f"Could not find metric [{metric_name}] in response:\n {response.text}" 56 | ) 57 | time.sleep(timeout) 58 | except ConnectionError: 59 | # NewConnectionError is expected until metrics are available 60 | time.sleep(10) 61 | 62 | exit(-1) 63 | -------------------------------------------------------------------------------- /tests/templates/kuttl/upgrade/test_nifi_metrics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import requests 4 | import time 5 | from requests.exceptions import ConnectionError 6 | 7 | if __name__ == "__main__": 8 | # Construct an argument parser 9 | all_args = argparse.ArgumentParser() 10 | # Add arguments to the parser 11 | all_args.add_argument( 12 | "-m", 13 | "--metric", 14 | required=False, 15 | default="nifi_amount_bytes_read", 16 | help="The name of a certain metric to check", 17 | ) 18 | all_args.add_argument( 19 | "-n", "--namespace", required=True, help="The namespace the test is running in" 20 | ) 21 | all_args.add_argument( 22 | "-p", 23 | "--port", 24 | required=False, 25 | default="8081", 26 | help="The port where metrics are exposed", 27 | ) 28 | all_args.add_argument( 29 | "-t", 30 | "--timeout", 31 | required=False, 32 | default="120", 33 | help="The timeout in seconds to wait for the metrics port to be opened", 34 | ) 35 | 36 | args = vars(all_args.parse_args()) 37 | metric_name = args["metric"] 38 | namespace = args["namespace"] 39 | port = args["port"] 40 | timeout = int(args["timeout"]) 41 | 42 | url = f"http://test-nifi-node-default-metrics.{namespace}.svc.cluster.local:{port}/metrics" 43 | 44 | # wait for 'timeout' seconds 45 | t_end = time.time() + timeout 46 | while time.time() < t_end: 47 | try: 48 | response = requests.post(url) 49 | response.raise_for_status() 50 | if metric_name in response.text: 51 | print("Test metrics succeeded!") 52 | exit(0) 53 | else: 54 | print( 55 | f"Could not find metric [{metric_name}] in response:\n {response.text}" 56 | ) 57 | time.sleep(timeout) 58 | except ConnectionError: 59 | # NewConnectionError is expected until metrics are available 60 | time.sleep(10) 61 | 62 | exit(-1) 63 | -------------------------------------------------------------------------------- /crate-hashes.json: -------------------------------------------------------------------------------- 1 | { 2 | "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-client@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", 3 | "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-core@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", 4 | "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-derive@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", 5 | "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-runtime@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", 6 | "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", 7 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#k8s-version@0.1.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", 8 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-operator-derive@0.3.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", 9 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-operator@0.100.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", 10 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-shared@0.0.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", 11 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-telemetry@0.6.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", 12 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-versioned-macros@0.8.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", 13 | "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-versioned@0.8.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", 14 | "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" 15 | } -------------------------------------------------------------------------------- /deny.toml: -------------------------------------------------------------------------------- 1 | # This file is the source of truth for all our repos! 2 | # This includes repos not templated by operator-templating, please copy/paste the file for this repos. 3 | 4 | # TIP: Use "cargo deny check" to check if everything is fine 5 | 6 | [graph] 7 | targets = [ 8 | { triple = "x86_64-unknown-linux-gnu" }, 9 | { triple = "aarch64-unknown-linux-gnu" }, 10 | { triple = "x86_64-unknown-linux-musl" }, 11 | { triple = "aarch64-apple-darwin" }, 12 | { triple = "x86_64-apple-darwin" }, 13 | ] 14 | 15 | [advisories] 16 | yanked = "deny" 17 | ignore = [ 18 | # https://rustsec.org/advisories/RUSTSEC-2023-0071 19 | # "rsa" crate: Marvin Attack: potential key recovery through timing sidechannel 20 | # 21 | # No patch is yet available, however work is underway to migrate to a fully constant-time implementation 22 | # So we need to accept this, as of SDP 25.3 we are not using the rsa crate to create certificates used in production 23 | # setups. 24 | # 25 | # https://github.com/RustCrypto/RSA/issues/19 is the tracking issue 26 | "RUSTSEC-2023-0071", 27 | ] 28 | 29 | [bans] 30 | multiple-versions = "allow" 31 | 32 | [licenses] 33 | unused-allowed-license = "allow" 34 | confidence-threshold = 1.0 35 | allow = [ 36 | "Apache-2.0", 37 | "BSD-2-Clause", 38 | "BSD-3-Clause", 39 | "CC0-1.0", 40 | "ISC", 41 | "LicenseRef-ring", 42 | "LicenseRef-webpki", 43 | "MIT", 44 | "MPL-2.0", 45 | "OpenSSL", # Needed for the ring and/or aws-lc-sys crate. See https://github.com/stackabletech/operator-templating/pull/464 for details 46 | "Unicode-3.0", 47 | "Unicode-DFS-2016", 48 | "Zlib", 49 | "Unlicense", 50 | ] 51 | private = { ignore = true } 52 | 53 | [[licenses.clarify]] 54 | name = "ring" 55 | expression = "LicenseRef-ring" 56 | license-files = [ 57 | { path = "LICENSE", hash = 0xbd0eed23 }, 58 | ] 59 | 60 | [[licenses.clarify]] 61 | name = "webpki" 62 | expression = "LicenseRef-webpki" 63 | license-files = [ 64 | { path = "LICENSE", hash = 0x001c7e6c }, 65 | ] 66 | 67 | [sources] 68 | unknown-registry = "deny" 69 | unknown-git = "deny" 70 | 71 | [sources.allow-org] 72 | github = ["stackabletech"] 73 | -------------------------------------------------------------------------------- /rust/operator-binary/src/security/tls.rs: -------------------------------------------------------------------------------- 1 | use snafu::{ResultExt, Snafu}; 2 | use stackable_operator::{ 3 | builder::pod::volume::{SecretFormat, SecretOperatorVolumeSourceBuilder, VolumeBuilder}, 4 | k8s_openapi::api::core::v1::Volume, 5 | shared::time::Duration, 6 | }; 7 | 8 | use crate::{crd::v1alpha1, security::authentication::STACKABLE_TLS_STORE_PASSWORD}; 9 | 10 | pub const KEYSTORE_VOLUME_NAME: &str = "keystore"; 11 | pub const KEYSTORE_NIFI_CONTAINER_MOUNT: &str = "/stackable/keystore"; 12 | pub const TRUSTSTORE_VOLUME_NAME: &str = "truststore"; 13 | 14 | type Result = std::result::Result; 15 | 16 | #[derive(Snafu, Debug)] 17 | pub enum Error { 18 | #[snafu(display("failed to build TLS certificate SecretClass Volume"))] 19 | TlsCertSecretClassVolumeBuild { 20 | source: stackable_operator::builder::pod::volume::SecretOperatorVolumeSourceBuilderError, 21 | }, 22 | } 23 | 24 | pub(crate) fn build_tls_volume( 25 | nifi: &v1alpha1::NifiCluster, 26 | volume_name: &str, 27 | service_scopes: impl IntoIterator>, 28 | secret_format: SecretFormat, 29 | requested_secret_lifetime: &Duration, 30 | listener_scope: Option<&str>, 31 | ) -> Result { 32 | let mut secret_volume_source_builder = 33 | SecretOperatorVolumeSourceBuilder::new(nifi.server_tls_secret_class()); 34 | 35 | if secret_format == SecretFormat::TlsPkcs12 { 36 | secret_volume_source_builder.with_tls_pkcs12_password(STACKABLE_TLS_STORE_PASSWORD); 37 | } 38 | for scope in service_scopes { 39 | secret_volume_source_builder.with_service_scope(scope.as_ref()); 40 | } 41 | if let Some(listener_scope) = listener_scope { 42 | secret_volume_source_builder.with_listener_volume_scope(listener_scope); 43 | } 44 | 45 | Ok(VolumeBuilder::new(volume_name) 46 | .ephemeral( 47 | secret_volume_source_builder 48 | .with_pod_scope() 49 | .with_format(secret_format) 50 | .with_auto_tls_cert_lifetime(*requested_secret_lifetime) 51 | .build() 52 | .context(TlsCertSecretClassVolumeBuildSnafu)?, 53 | ) 54 | .build()) 55 | } 56 | -------------------------------------------------------------------------------- /tests/templates/kuttl/smoke_v2/50-install-test-nifi.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: test-nifi 6 | labels: 7 | app: test-nifi 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: test-nifi 13 | template: 14 | metadata: 15 | labels: 16 | app: test-nifi 17 | spec: 18 | serviceAccountName: "tls-sa" 19 | securityContext: 20 | fsGroup: 1000 21 | containers: 22 | - name: test-nifi 23 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 24 | command: ["sleep", "infinity"] 25 | resources: 26 | requests: 27 | memory: "128Mi" 28 | cpu: "100m" 29 | limits: 30 | memory: "128Mi" 31 | cpu: "400m" 32 | volumeMounts: 33 | - name: tls 34 | mountPath: /stackable/tls 35 | volumes: 36 | - name: tls 37 | ephemeral: 38 | volumeClaimTemplate: 39 | metadata: 40 | annotations: 41 | secrets.stackable.tech/class: tls 42 | secrets.stackable.tech/scope: pod 43 | spec: 44 | storageClassName: secrets.stackable.tech 45 | accessModes: 46 | - ReadWriteOnce 47 | resources: 48 | requests: 49 | storage: "1" 50 | --- 51 | apiVersion: v1 52 | kind: ServiceAccount 53 | metadata: 54 | name: tls-sa 55 | {% if test_scenario['values']['openshift'] == 'true' %} 56 | --- 57 | kind: Role 58 | apiVersion: rbac.authorization.k8s.io/v1 59 | metadata: 60 | name: use-integration-tests-scc 61 | rules: 62 | - apiGroups: ["security.openshift.io"] 63 | resources: ["securitycontextconstraints"] 64 | resourceNames: ["privileged"] 65 | verbs: ["use"] 66 | --- 67 | kind: RoleBinding 68 | apiVersion: rbac.authorization.k8s.io/v1 69 | metadata: 70 | name: use-integration-tests-scc 71 | subjects: 72 | - kind: ServiceAccount 73 | name: tls-sa 74 | roleRef: 75 | kind: Role 76 | name: use-integration-tests-scc 77 | apiGroup: rbac.authorization.k8s.io 78 | {% endif %} 79 | -------------------------------------------------------------------------------- /docs/modules/nifi/pages/getting_started/installation.adoc: -------------------------------------------------------------------------------- 1 | = Installation 2 | :description: Install the Stackable operator for Apache NiFi and its dependencies using stackablectl or Helm. Follow steps for a complete setup on Kubernetes. 3 | 4 | On this page you install the Stackable operator for Apache NiFi as well as the commons, secret and listener operator, 5 | which are required by all Stackable operators. 6 | 7 | There are multiple ways to install the Stackable operators. 8 | xref:management:stackablectl:index.adoc[] is the preferred way, but Helm is also supported. 9 | OpenShift users may prefer installing the operator from the RedHat Certified Operator catalog using the OpenShift web console. 10 | 11 | [tabs] 12 | ==== 13 | stackablectl:: 14 | + 15 | -- 16 | The `stackablectl` command line tool is the recommended way to interact with operators and dependencies. 17 | Follow the xref:management:stackablectl:installation.adoc[installation steps] for your platform if you choose to work with `stackablectl`. 18 | 19 | After you have installed `stackablectl` and have a Kubernetes cluster up and running, run the following command to 20 | install all operators necessary for NiFi: 21 | 22 | [source,shell] 23 | ---- 24 | include::example$getting_started/getting_started.sh[tag=stackablectl-install-operators] 25 | ---- 26 | 27 | The output should contain the following lines 28 | 29 | [source] 30 | include::example$getting_started/install_output.txt[] 31 | 32 | TIP: Consult the xref:management:stackablectl:quickstart.adoc[] to learn more about how to use `stackablectl`. 33 | -- 34 | 35 | Helm:: 36 | + 37 | -- 38 | You can also use Helm to install the operators. 39 | 40 | NOTE: `helm repo` subcommands are not supported for OCI registries. The operators are installed directly, without adding the Helm Chart repository first. 41 | 42 | Install the Stackable operators: 43 | 44 | [source,bash] 45 | ---- 46 | include::example$getting_started/getting_started.sh[tag=helm-install-operators] 47 | ---- 48 | 49 | Helm deploys the operators in a Kubernetes Deployment and applies the CRDs for the Apache NiFi service (as well as the CRDs for the required operators). 50 | -- 51 | ==== 52 | 53 | == What's next 54 | 55 | xref:getting_started/first_steps.adoc[Setting up a NiFi cluster] and its dependencies. 56 | -------------------------------------------------------------------------------- /tests/templates/kuttl/oidc-opa/41-install-test-container.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: python 6 | --- 7 | kind: Role 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | metadata: 10 | name: python 11 | {% if test_scenario['values']['openshift'] == 'true' %} 12 | rules: 13 | - apiGroups: ["security.openshift.io"] 14 | resources: ["securitycontextconstraints"] 15 | resourceNames: ["privileged"] 16 | verbs: ["use"] 17 | {% endif %} 18 | --- 19 | kind: RoleBinding 20 | apiVersion: rbac.authorization.k8s.io/v1 21 | metadata: 22 | name: python 23 | subjects: 24 | - kind: ServiceAccount 25 | name: python 26 | roleRef: 27 | kind: Role 28 | name: python 29 | apiGroup: rbac.authorization.k8s.io 30 | --- 31 | apiVersion: kuttl.dev/v1beta1 32 | kind: TestStep 33 | metadata: 34 | name: install-test-container 35 | timeout: 300 36 | --- 37 | apiVersion: apps/v1 38 | kind: StatefulSet 39 | metadata: 40 | name: python 41 | labels: 42 | app: python 43 | spec: 44 | replicas: 1 45 | selector: 46 | matchLabels: 47 | app: python 48 | template: 49 | metadata: 50 | labels: 51 | app: python 52 | spec: 53 | serviceAccountName: python 54 | containers: 55 | - name: oidc-login-test 56 | image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev 57 | stdin: true 58 | tty: true 59 | resources: 60 | requests: 61 | memory: "128Mi" 62 | cpu: "512m" 63 | limits: 64 | memory: "128Mi" 65 | cpu: "1" 66 | env: 67 | - name: NAMESPACE 68 | valueFrom: 69 | fieldRef: 70 | fieldPath: metadata.namespace 71 | - name: NIFI_VERSION 72 | value: "{{ test_scenario['values']['nifi'] }}" 73 | - name: OIDC_USE_TLS 74 | value: "{{ test_scenario['values']['oidc-use-tls'] }}" 75 | volumeMounts: 76 | - name: test-script 77 | mountPath: /tmp/test-script 78 | terminationGracePeriodSeconds: 1 79 | volumes: 80 | - name: test-script 81 | configMap: 82 | name: test-script 83 | terminationGracePeriodSeconds: 1 84 | --------------------------------------------------------------------------------