├── roles ├── redpanda_console │ ├── README.md │ ├── tests │ │ ├── requirements.txt │ │ ├── defaults_test.py │ │ └── pre_v3_defaults_test.py │ ├── templates │ │ ├── console.yml │ │ ├── defaults.j2 │ │ └── pre_v3_defaults.j2 │ ├── Dockerfile │ ├── tasks │ │ ├── start-console.yml │ │ ├── create-user.yml │ │ ├── install-console-rpm.yml │ │ ├── install-console-deb.yml │ │ ├── configure-deb-repository.yml │ │ ├── configure-console.yml │ │ ├── install-certs.yml │ │ ├── main.yml │ │ └── configure-rpm-repository.yml │ ├── docker-compose.yml │ ├── vars │ │ └── main.yml │ ├── Makefile │ └── defaults │ │ └── main.yml ├── redpanda_connect │ ├── tests │ │ ├── requirements.txt │ │ ├── jmx-exporter-config_test.py │ │ ├── redpanda-connect.service_test.py │ │ └── connect-distributed.properties_test.py │ ├── Dockerfile │ ├── docker-compose.yml │ ├── templates │ │ ├── logging.properties.j2 │ │ ├── java-home.sh.j2 │ │ ├── log4j.properties.j2 │ │ ├── connect-distributed │ │ │ ├── connect-distributed.properties.j2 │ │ │ └── connect-distributed-tls.properties.j2 │ │ └── redpanda-connect.service.j2 │ ├── tasks │ │ ├── install-connect.yml │ │ ├── generate-java-home.yml │ │ ├── create-user.yml │ │ ├── generate-unit-file.yml │ │ ├── copy-truststore.yml │ │ ├── safe-restart.yml │ │ ├── generate-log4j-config.yml │ │ ├── copy-keystore.yml │ │ ├── generate-connect-distributed.yml │ │ ├── install-certs.yml │ │ ├── generate-jmx-exporter-config.yml │ │ └── main.yml │ ├── Makefile │ ├── README.md │ └── defaults │ │ └── main.yml ├── redpanda_broker │ ├── templates │ │ ├── redpanda.yml │ │ ├── bootstrap.yml │ │ └── configs │ │ │ ├── fips.j2 │ │ │ ├── tiered_storage.j2 │ │ │ ├── tls.j2 │ │ │ └── defaults.j2 │ ├── tests │ │ ├── requirements.txt │ │ ├── requirements.yml │ │ ├── mocks │ │ │ └── mock_ansible_module.py │ │ ├── restart_required.yml │ │ ├── defaults_test.py │ │ ├── tiered-storage_test.py │ │ ├── restart_required_test.py │ │ └── tls_test.py │ ├── docker-compose.yml │ ├── vars │ │ └── main.yml │ ├── Dockerfile │ ├── Makefile │ ├── tasks │ │ ├── fips-assert.yml │ │ ├── install-nightly-build-rpm.yml │ │ ├── install-nightly-build-deb.yml │ │ ├── safe-restart.yml │ │ ├── install-certs.yml │ │ ├── configure-deb-repository.yml │ │ ├── install-rp-deb.yml │ │ ├── install-rp-rpm.yml │ │ ├── install-rp-rpm-airgap.yml │ │ ├── main.yml │ │ ├── install-rp-deb-airgap.yml │ │ └── configure-rpm-repository.yml │ ├── library │ │ └── mock_ansible_module.py │ ├── README.md │ └── defaults │ │ └── main.yml ├── redpanda_logging │ ├── tests │ │ ├── requirements.txt │ │ └── rsyslog_test.py │ ├── templates │ │ ├── redpanda-rsyslog.conf.j2 │ │ └── redpanda-logrotate.conf.j2 │ ├── docker-compose.yml │ ├── Dockerfile │ ├── handlers │ │ └── main.yml │ ├── Makefile │ ├── meta │ │ └── main.yml │ ├── defaults │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ └── README.md ├── binary_bundler │ ├── README.md │ ├── tasks │ │ ├── deb_bundle.yml │ │ ├── main.yml │ │ └── rpm_bundle.yml │ └── defaults │ │ └── main.yml ├── demo_certs │ ├── README.md │ ├── templates │ │ ├── node.conf.tpl │ │ ├── connect.conf.tpl │ │ └── ca.conf.tpl │ ├── defaults │ │ └── main.yml │ └── tasks │ │ ├── generate-truststore.yml │ │ ├── create-ca.yml │ │ ├── main.yml │ │ ├── issue-certs.yml │ │ ├── issue-certs-keystore.yml │ │ ├── generate-csrs.yml │ │ └── generate-csrs-keystore.yml ├── client_config │ ├── tasks │ │ ├── main.yml │ │ ├── install-cert.yml │ │ └── install-rpk.yml │ ├── README.md │ └── defaults │ │ └── main.yml ├── system_setup │ ├── templates │ │ ├── dnf_proxy.j2 │ │ └── apt_proxy.j2 │ ├── tasks │ │ ├── data-dir-perms.yml │ │ ├── create-redpanda-user.yml │ │ ├── install-node-deps-deb.yml │ │ ├── install-node-deps-rpm.yml │ │ ├── main.yml │ │ └── prepare-data-dir.yml │ ├── README.md │ └── defaults │ │ └── main.yml └── sysctl_setup │ ├── README.md │ ├── tasks │ └── main.yml │ └── defaults │ └── main.yml ├── .idea ├── vcs.xml ├── .gitignore ├── inspectionProfiles │ └── profiles_settings.xml ├── modules.xml ├── redpanda-ansible-collection.iml └── misc.xml ├── .gitignore ├── .ansible-lint ├── .buildkite └── pipeline.yml ├── meta └── runtime.yml ├── galaxy.yml ├── README.md └── Makefile /roles/redpanda_console/README.md: -------------------------------------------------------------------------------- 1 | # Ansible Deployment for Redpanda Console 2 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tests/requirements.txt: -------------------------------------------------------------------------------- 1 | jinja2==3.1.3 2 | pyyaml==6.0.1 3 | -------------------------------------------------------------------------------- /roles/redpanda_console/tests/requirements.txt: -------------------------------------------------------------------------------- 1 | jinja2==3.1.3 2 | pyyaml==6.0.1 3 | -------------------------------------------------------------------------------- /roles/redpanda_broker/templates/redpanda.yml: -------------------------------------------------------------------------------- 1 | {{ configuration.node | to_nice_yaml }} 2 | -------------------------------------------------------------------------------- /roles/redpanda_console/templates/console.yml: -------------------------------------------------------------------------------- 1 | {{ console_config | to_nice_yaml }} 2 | -------------------------------------------------------------------------------- /roles/redpanda_broker/templates/bootstrap.yml: -------------------------------------------------------------------------------- 1 | {{ configuration.cluster | to_nice_yaml }} 2 | -------------------------------------------------------------------------------- /roles/redpanda_logging/tests/requirements.txt: -------------------------------------------------------------------------------- 1 | pytest==7.4.0 2 | pyyaml==6.0.1 3 | jinja2==3.1.2 -------------------------------------------------------------------------------- /roles/binary_bundler/README.md: -------------------------------------------------------------------------------- 1 | ## Bindary Bundler for Redpanda 2 | 3 | Downloads and bundles binaries for use in installing a Redpanda Cluster 4 | -------------------------------------------------------------------------------- /roles/redpanda_logging/templates/redpanda-rsyslog.conf.j2: -------------------------------------------------------------------------------- 1 | if $programname == '{{ redpanda_logging_program }}' then {{ redpanda_logging_log_file }} 2 | & stop -------------------------------------------------------------------------------- /roles/redpanda_broker/tests/requirements.txt: -------------------------------------------------------------------------------- 1 | jinja2>=3.1.3 2 | pyyaml>=6.0.1 3 | ansible>=8.3.0 4 | pytest>=7.4.2 5 | ansible-core>=2.15.5 6 | ansible-runner>=2.3.2 7 | -------------------------------------------------------------------------------- /roles/redpanda_connect/Dockerfile: -------------------------------------------------------------------------------- 1 | ## Used for testing the Kafka Connect role 2 | FROM python:latest 3 | 4 | WORKDIR /app 5 | 6 | COPY . . 7 | 8 | CMD ["make", "do"] 9 | -------------------------------------------------------------------------------- /roles/redpanda_console/Dockerfile: -------------------------------------------------------------------------------- 1 | ## Used for testing the Kafka Connect role 2 | FROM python:latest 3 | 4 | WORKDIR /app 5 | 6 | COPY . . 7 | 8 | CMD ["make", "do"] 9 | -------------------------------------------------------------------------------- /roles/demo_certs/README.md: -------------------------------------------------------------------------------- 1 | # Ansible Collection - redpanda.cluster.demo_certs 2 | 3 | Allows the user to generate a local CA with certs to consume in building the cluster for demo purposes only 4 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /roles/client_config/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install cert 3 | ansible.builtin.include_tasks: install-cert.yml 4 | 5 | - name: Install rpk 6 | ansible.builtin.include_tasks: install-rpk.yml 7 | -------------------------------------------------------------------------------- /roles/system_setup/templates/dnf_proxy.j2: -------------------------------------------------------------------------------- 1 | [main] 2 | gpgcheck=1 3 | installonly_limit=3 4 | clean_requirements_on_remove=True 5 | best=False 6 | skip_if_unavailable=True 7 | proxy=http://{{ rpm_proxy }} 8 | -------------------------------------------------------------------------------- /roles/redpanda_logging/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | testctr: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile 7 | volumes: 8 | - .:/app 9 | command: make do -------------------------------------------------------------------------------- /roles/redpanda_console/tasks/start-console.yml: -------------------------------------------------------------------------------- 1 | # ensure that systemd starts up redpanda console 2 | - name: Start redpanda console 3 | ansible.builtin.systemd: 4 | name: redpanda-console 5 | state: restarted 6 | -------------------------------------------------------------------------------- /roles/system_setup/templates/apt_proxy.j2: -------------------------------------------------------------------------------- 1 | Acquire::http::Proxy "http://{{ https_proxy_value }}"; 2 | Acquire::https::Proxy "http://{{ https_proxy_value }}"; 3 | Acquire::ftp::Proxy "ftp://{{ https_proxy_value }}"; 4 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /roles/redpanda_broker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | testctr: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile 7 | volumes: 8 | - .:/app 9 | command: make do 10 | -------------------------------------------------------------------------------- /roles/redpanda_connect/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | testctr: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile 7 | volumes: 8 | - .:/app 9 | command: make do 10 | -------------------------------------------------------------------------------- /roles/redpanda_console/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | testctr: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile 7 | volumes: 8 | - .:/app 9 | command: make do 10 | -------------------------------------------------------------------------------- /roles/redpanda_connect/templates/logging.properties.j2: -------------------------------------------------------------------------------- 1 | handlers=java.util.logging.ConsoleHandler 2 | java.util.logging.ConsoleHandler.level=ALL 3 | io.prometheus.jmx.level=ALL 4 | io.prometheus.jmx.shaded.io.prometheus.jmx.level=ALL 5 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.gz 3 | *.xml 4 | *.iml 5 | .idea/redpanda-ansible-collection.iml 6 | .idea/misc.xml 7 | *.pyc 8 | 9 | .idea/** 10 | .idea/misc.xml 11 | .idea/redpanda-ansible-collection.iml 12 | roles/redpanda_broker/tests/inventory 13 | -------------------------------------------------------------------------------- /roles/sysctl_setup/README.md: -------------------------------------------------------------------------------- 1 | ## Sysctl for Redpanda Clusters 2 | 3 | Handles configuration of Sysctl prerequisites for Redpanda clusters 4 | 5 | If you handle this with a prebuilt node image and/or internal tooling you don't need to include this role in any plays. 6 | -------------------------------------------------------------------------------- /roles/redpanda_console/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | console_config_templates: 3 | - template: pre_v3_defaults.j2 4 | condition: "{{ use_pre_v3_template | default(false) }}" 5 | - template: defaults.j2 6 | condition: "{{ not (use_pre_v3_template | default(false)) }}" 7 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/install-connect.yml: -------------------------------------------------------------------------------- 1 | - name: Install Redpanda RPMs 2 | ansible.builtin.dnf: 3 | name: "{{ redpanda_connect_rpm_dir }}/{{ redpanda_connect_rpm }}" 4 | state: present 5 | disable_gpg_check: true 6 | register: rpm_result 7 | become: true 8 | -------------------------------------------------------------------------------- /roles/client_config/README.md: -------------------------------------------------------------------------------- 1 | # Ansible Collection - redpanda.cluster.client_config 2 | 3 | Handles creation of a lightweight client instance for use with RPK. 4 | 5 | While it does download RPK from github, this can be changed to pull from a within network re-host by setting rpk_url. 6 | -------------------------------------------------------------------------------- /roles/client_config/defaults/main.yml: -------------------------------------------------------------------------------- 1 | rpk_base_url: "https://github.com/redpanda-data/redpanda/releases/latest/download" 2 | cert_dest_dir: "/opt/rpk/certs" 3 | cert_src_dir: "{{ playbook_dir }}/tls/ca" 4 | 5 | # truststore used to auth to RP brokers with RPK 6 | truststore_name: "ca.crt" 7 | -------------------------------------------------------------------------------- /roles/redpanda_broker/templates/configs/fips.j2: -------------------------------------------------------------------------------- 1 | { 2 | "node": { 3 | "redpanda": { 4 | "fips_mode": "{{ fips_mode }}", 5 | "openssl_config_file": "{{ fips_openssl_config_file }}", 6 | "openssl_module_directory": "{{ fips_openssl_module_directory }}" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/generate-java-home.yml: -------------------------------------------------------------------------------- 1 | - name: Instantiate java-home.sh file 2 | ansible.builtin.template: 3 | src: java-home.sh.j2 4 | dest: /etc/profile.d/java-home.sh 5 | owner: "{{ redpanda_user }}" 6 | group: "{{ redpanda_group }}" 7 | mode: '0755' 8 | force: true 9 | -------------------------------------------------------------------------------- /roles/redpanda_connect/templates/java-home.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export JAVA_HOME={{ java_home }} 4 | export PATH=$JAVA_HOME/bin:$PATH 5 | export CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/tools.jar 6 | export CONNECT_HOME={{ redpanda_connect_home }} 7 | 8 | {{ java_home_extra_content | default('') }} 9 | -------------------------------------------------------------------------------- /roles/system_setup/tasks/data-dir-perms.yml: -------------------------------------------------------------------------------- 1 | - name: Set data dir permissions 2 | ansible.builtin.file: 3 | dest: "{{ redpanda_base_dir }}" 4 | src: "{{ repdanda_mount_dir }}" 5 | state: link 6 | owner: redpanda 7 | group: redpanda 8 | mode: "0755" 9 | tags: 10 | - data_dir_perms 11 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/redpanda-ansible-collection.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /roles/system_setup/tasks/create-redpanda-user.yml: -------------------------------------------------------------------------------- 1 | - name: Create Redpanda group 2 | group: 3 | name: redpanda 4 | state: present 5 | system: yes 6 | become: yes 7 | 8 | - name: Create Redpanda user 9 | user: 10 | name: redpanda 11 | group: redpanda 12 | state: present 13 | system: yes 14 | become: yes 15 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | -------------------------------------------------------------------------------- /.ansible-lint: -------------------------------------------------------------------------------- 1 | --- 2 | exclude_paths: 3 | - .github/** 4 | - Taskfile.yaml 5 | - tests/** 6 | 7 | enable_list: 8 | - yaml 9 | - empty-string-compare 10 | - args 11 | 12 | profile: production 13 | 14 | skip_list: 15 | - risky-shell-pipe 16 | - jinja[spacing] 17 | - yaml[brackets] 18 | - yaml[line-length] 19 | - yaml[trailing-spaces] 20 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tests/requirements.yml: -------------------------------------------------------------------------------- 1 | collections: 2 | - name: community.general 3 | - name: redpanda.cluster 4 | type: galaxy 5 | - name: ansible.posix 6 | - name: grafana.grafana 7 | - name: prometheus.prometheus 8 | 9 | roles: 10 | - src: mrlesmithjr.mdadm 11 | - src: mrlesmithjr.squid 12 | - src: geerlingguy.node_exporter 13 | -------------------------------------------------------------------------------- /roles/demo_certs/templates/node.conf.tpl: -------------------------------------------------------------------------------- 1 | # OpenSSL node configuration file 2 | [ req ] 3 | prompt=no 4 | distinguished_name = distinguished_name 5 | req_extensions = extensions 6 | 7 | [ distinguished_name ] 8 | organizationName = Vectorized 9 | 10 | [ extensions ] 11 | subjectAltName = critical,DNS:{{ansible_hostname}},DNS:{{ansible_fqdn}},IP:{{inventory_hostname}},IP:{{private_ip}} 12 | -------------------------------------------------------------------------------- /roles/client_config/tasks/install-cert.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure certificate directory exists 3 | ansible.builtin.file: 4 | path: "{{ cert_dest_dir }}" 5 | state: directory 6 | mode: '0755' 7 | 8 | - name: Copy the certificate for the application 9 | ansible.builtin.copy: 10 | src: "{{ cert_src_dir}}/{{truststore_name}}" 11 | dest: "{{ cert_dest_dir }}/{{truststore_name}}" 12 | mode: '0644' 13 | -------------------------------------------------------------------------------- /roles/redpanda_broker/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | custom_config_templates: 3 | - template: configs/defaults.j2 4 | - template: configs/tls.j2 5 | condition: "{{ enable_tls | default(False) | bool }}" 6 | - template: configs/tiered_storage.j2 7 | condition: "{{ tiered_storage_bucket_name is defined | default(False) | bool }}" 8 | - template: configs/fips.j2 9 | condition: "{{ enable_fips | default(False) | bool}}" 10 | -------------------------------------------------------------------------------- /roles/redpanda_logging/Dockerfile: -------------------------------------------------------------------------------- 1 | # Used for testing Redpanda Roles 2 | FROM python:latest 3 | 4 | WORKDIR /app 5 | 6 | # Copy requirements first (for better caching) 7 | COPY tests/requirements.txt /app/requirements.txt 8 | 9 | # Install Python dependencies 10 | RUN pip install --no-cache-dir -r requirements.txt 11 | 12 | # Copy the entire role 13 | COPY . . 14 | 15 | # Run make do when container launches 16 | CMD ["make", "do"] -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/create-user.yml: -------------------------------------------------------------------------------- 1 | - name: Create redpanda group 2 | ansible.builtin.group: 3 | name: "{{ redpanda_group }}" 4 | state: present 5 | system: true 6 | become: true 7 | 8 | - name: Create redpanda user 9 | ansible.builtin.user: 10 | name: "{{ redpanda_user }}" 11 | group: "{{ redpanda_group }}" 12 | system: true 13 | shell: /usr/sbin/nologin 14 | create_home: false 15 | state: present 16 | become: true 17 | -------------------------------------------------------------------------------- /roles/redpanda_console/tasks/create-user.yml: -------------------------------------------------------------------------------- 1 | - name: Create redpanda group 2 | ansible.builtin.group: 3 | name: "{{ redpanda_group }}" 4 | state: present 5 | system: true 6 | become: true 7 | 8 | - name: Create redpanda user 9 | ansible.builtin.user: 10 | name: "{{ redpanda_user }}" 11 | group: "{{ redpanda_group }}" 12 | system: true 13 | shell: /usr/sbin/nologin 14 | create_home: false 15 | state: present 16 | become: true 17 | -------------------------------------------------------------------------------- /roles/system_setup/README.md: -------------------------------------------------------------------------------- 1 | ## System Setup for Redpanda Clusters 2 | 3 | Handles system configuration on individual brokers for a Redpanda cluster. This entails: 4 | * copying over certs 5 | * handling the configuration of the data directory 6 | * ensuring the data directory has correct permissions 7 | * installing dependencies on the node required for Redpanda to run 8 | 9 | If you handle this with a prebuilt node image and/or internal tooling you don't need to include this role in any plays. 10 | -------------------------------------------------------------------------------- /roles/sysctl_setup/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Max user instances 2 | ansible.posix.sysctl: 3 | name: fs.inotify.max_user_instances 4 | value: "{{ max_user_instances }}" 5 | sysctl_set: true 6 | state: present 7 | reload: true 8 | become: true 9 | 10 | - name: Set kernel panic on oops 11 | ansible.posix.sysctl: 12 | name: kernel.panic_on_oops 13 | value: "{{ kernel_panic_on_oops }}" 14 | sysctl_set: true 15 | state: present 16 | reload: true 17 | become: true 18 | -------------------------------------------------------------------------------- /roles/demo_certs/templates/connect.conf.tpl: -------------------------------------------------------------------------------- 1 | # OpenSSL node configuration file for keystore generation 2 | [ req ] 3 | prompt = no 4 | distinguished_name = distinguished_name 5 | x509_extensions = extensions 6 | 7 | [ distinguished_name ] 8 | organizationName = Vectorized 9 | commonName = {{ ansible_hostname }} 10 | 11 | [ extensions ] 12 | basicConstraints = CA:FALSE 13 | keyUsage = critical, digitalSignature, keyEncipherment 14 | extendedKeyUsage = serverAuth, clientAuth 15 | subjectAltName = DNS:{{ ansible_hostname }},DNS:{{ ansible_fqdn }},IP:{{ inventory_hostname }},IP:{{ private_ip }} 16 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/generate-unit-file.yml: -------------------------------------------------------------------------------- 1 | - name: Instantiate systemd unit file for kafka connect 2 | ansible.builtin.template: 3 | src: redpanda-connect.service.j2 4 | dest: "/etc/systemd/system/redpanda-connect.service" 5 | owner: "root" 6 | group: "root" 7 | mode: '0644' 8 | become: true 9 | register: systemd_unit_result 10 | 11 | - name: Chown /opt/kafka 12 | ansible.builtin.file: 13 | path: /opt/kafka 14 | owner: "{{ redpanda_user }}" 15 | group: "{{ redpanda_group }}" 16 | state: directory 17 | recurse: true 18 | become: true 19 | -------------------------------------------------------------------------------- /roles/redpanda_logging/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Handlers for redpanda_logging role 3 | 4 | - name: restart rsyslog 5 | ansible.builtin.systemd: 6 | name: rsyslog 7 | state: restarted 8 | become: true 9 | 10 | - name: reload systemd 11 | ansible.builtin.systemd: 12 | daemon_reload: true 13 | become: true 14 | 15 | - name: restart redpanda services 16 | ansible.builtin.systemd: 17 | name: "{{ item }}" 18 | state: restarted 19 | become: true 20 | loop: 21 | - redpanda 22 | - redpanda-connect 23 | - redpanda-console 24 | failed_when: false # Don't fail if service doesn't exist -------------------------------------------------------------------------------- /roles/system_setup/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | redpanda_base_dir: /var/lib/redpanda 3 | repdanda_mount_dir: /mnt/vectorized/redpanda 4 | redpanda_truststore_file: "{{ redpanda_certs_dir }}/truststore.pem" 5 | redpanda_cert_file: "{{ redpanda_certs_dir }}/node.crt" 6 | debian_prerequisite_packages: 7 | - debian-keyring 8 | - debian-archive-keyring 9 | - apt-transport-https 10 | - ca-certificates 11 | - gnupg 12 | - iotop 13 | - mdadm 14 | - xfsprogs 15 | - openssl 16 | rpm_prerequisite_packages: 17 | - curl 18 | - yum-utils 19 | - dnf-plugins-core 20 | - openssl 21 | redpanda_certs_dir: /etc/redpanda/certs 22 | -------------------------------------------------------------------------------- /roles/demo_certs/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | redpanda_organization: redpanda-test 3 | redpanda_cluster_id: redpanda 4 | redpanda_certs_dir: /etc/redpanda/certs 5 | redpanda_csr_file: "{{ redpanda_certs_dir }}/node.csr" 6 | redpanda_key_file: "{{ redpanda_certs_dir }}/node.key" 7 | redpanda_cert_file: "{{ redpanda_certs_dir }}/node.crt" 8 | redpanda_truststore_file: "{{ redpanda_certs_dir }}/truststore.pem" 9 | truststore_password: "password" 10 | keystore_password: "password" 11 | redpanda_truststores_dir: /etc/redpanda/truststores 12 | 13 | truststore_type: PKCS12 14 | keystore_file_name: keystore.p12 15 | keystores_type: PKCS12 16 | redpanda_user: redpanda 17 | redpanda_group: redpanda 18 | -------------------------------------------------------------------------------- /roles/redpanda_logging/Makefile: -------------------------------------------------------------------------------- 1 | PYTHON ?= python 2 | PIP ?= pip 3 | 4 | DIR := tests 5 | SCRIPTS := $(wildcard $(DIR)/*_test.py) 6 | 7 | .PHONY: do 8 | do: reqs test 9 | 10 | .PHONY: test 11 | test: $(SCRIPTS) 12 | @echo "running $(DIR) $(SCRIPTS)" 13 | @EXIT_CODE=0; \ 14 | for FILE in $(SCRIPTS); do \ 15 | echo "Running $$FILE"; \ 16 | $(PYTHON) $$FILE || EXIT_CODE=1; \ 17 | done; \ 18 | exit $$EXIT_CODE 19 | 20 | .PHONY: reqs 21 | reqs: 22 | $(PIP) install --no-cache-dir -r tests/requirements.txt 23 | 24 | .PHONY: up 25 | up: 26 | docker build -t redpanda-logging-test . && docker run redpanda-logging-test 27 | 28 | .PHONY: compose-up 29 | compose-up: 30 | docker compose up --build -------------------------------------------------------------------------------- /roles/redpanda_broker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Used for testing Redpanda Roles 2 | FROM python:latest 3 | 4 | WORKDIR /app 5 | 6 | # Copy the current directory contents into the container at /app 7 | COPY tests/requirements.txt /app/requirements.txt 8 | 9 | # Install Ansible and other dependencies 10 | RUN pip install --no-cache-dir -r requirements.txt 11 | 12 | COPY tests/requirements.yml /app/requirements.yml 13 | RUN ansible-galaxy collection install -r requirements.yml --force 14 | RUN ansible-galaxy role install -r requirements.yml --force 15 | 16 | COPY . . 17 | 18 | # Set ANSIBLE_LIBRARY environment variable 19 | ENV ANSIBLE_LIBRARY=/app/library 20 | 21 | # Run make do when the container launches 22 | CMD ["make", "do"] 23 | -------------------------------------------------------------------------------- /roles/redpanda_connect/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test reqs do 2 | 3 | PYTHON ?= python 4 | PIP ?= pip 5 | 6 | DIR := tests 7 | SCRIPTS := $(wildcard $(DIR)/*_test.py) 8 | 9 | # Target to process each file 10 | all: $(FILES) 11 | 12 | do: reqs test 13 | 14 | .PHONY: test 15 | test: $(SCRIPTS) 16 | @echo "running $(DIR) $(SCRIPTS)" 17 | @EXIT_CODE=0; \ 18 | for FILE in $(SCRIPTS); do \ 19 | echo "Running $$FILE"; \ 20 | $(PYTHON) $$FILE || EXIT_CODE=1; \ 21 | done; \ 22 | exit $$EXIT_CODE 23 | reqs: 24 | $(PIP) install --no-cache-dir -r tests/requirements.txt 25 | 26 | .PHONY: up 27 | up: 28 | docker build -t redpanda-connect-test . && docker run redpanda-connect-test 29 | 30 | .PHONY: compose-up 31 | compose-up: 32 | docker compose up --build 33 | -------------------------------------------------------------------------------- /roles/redpanda_console/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test reqs do 2 | 3 | PYTHON ?= python 4 | PIP ?= pip 5 | 6 | DIR := tests 7 | SCRIPTS := $(wildcard $(DIR)/*_test.py) 8 | 9 | # Target to process each file 10 | all: $(FILES) 11 | 12 | do: reqs test 13 | 14 | .PHONY: test 15 | test: $(SCRIPTS) 16 | @echo "running $(DIR) $(SCRIPTS)" 17 | @EXIT_CODE=0; \ 18 | for FILE in $(SCRIPTS); do \ 19 | echo "Running $$FILE"; \ 20 | $(PYTHON) $$FILE || EXIT_CODE=1; \ 21 | done; \ 22 | exit $$EXIT_CODE 23 | reqs: 24 | $(PIP) install --no-cache-dir -r tests/requirements.txt 25 | 26 | .PHONY: up 27 | up: 28 | docker build -t redpanda-console-test . && docker run redpanda-console-test 29 | 30 | .PHONY: compose-up 31 | compose-up: 32 | docker compose up --build 33 | -------------------------------------------------------------------------------- /roles/sysctl_setup/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # inotify.max_user_instances caps the number of inotify watches a specific user 2 | # is allowed to create in Linux. In some larger setups, Redpanda may need to create 3 | # more. If this is set too low (like 512), it could manifest as odd behavior in 4 | # Schema Registry. 5 | # 6 | # Adjust upwards as necessary. An upper limit has not been established. Increasing 7 | # this excessively can increase kernel memory usage. 8 | max_user_instances: 8192 9 | 10 | # kernel.panic_on_oops controls whether the kernel will panic when it detects 11 | # an oops (kernel error). Setting to 0 means the kernel will try to continue 12 | # running after an oops. Setting to 1 causes an immediate kernel panic. 13 | kernel_panic_on_oops: 1 14 | -------------------------------------------------------------------------------- /roles/redpanda_connect/templates/log4j.properties.j2: -------------------------------------------------------------------------------- 1 | log4j.rootLogger={{ log4j_log_level }}, stdout, syslogAppender 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | 6 | log4j.appender.syslogAppender=org.apache.log4j.net.SyslogAppender 7 | log4j.appender.syslogAppender.syslogHost=localhost 8 | log4j.appender.syslogAppender.facility=LOCAL0 9 | log4j.appender.syslogAppender.layout=org.apache.log4j.PatternLayout 10 | 11 | connect.log.pattern=[%d] %p %X{connector.context}%m (%c:%L)%n 12 | 13 | log4j.appender.stdout.layout.ConversionPattern=${connect.log.pattern} 14 | log4j.appender.syslogAppender.layout.ConversionPattern=${connect.log.pattern} 15 | 16 | log4j.logger.org.reflections=ERROR 17 | -------------------------------------------------------------------------------- /roles/redpanda_logging/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: Redpanda Data 4 | description: Configure logging for Redpanda services to dedicated log files with rotation 5 | company: Redpanda Data 6 | license: Apache-2.0 7 | min_ansible_version: 2.9 8 | platforms: 9 | - name: Ubuntu 10 | versions: 11 | - focal 12 | - jammy 13 | - name: Debian 14 | versions: 15 | - buster 16 | - bullseye 17 | - name: EL 18 | versions: 19 | - 7 20 | - 8 21 | - 9 22 | - name: Fedora 23 | versions: 24 | - all 25 | galaxy_tags: 26 | - redpanda 27 | - logging 28 | - syslog 29 | - logrotate 30 | - kafka 31 | - streaming 32 | 33 | dependencies: [] -------------------------------------------------------------------------------- /roles/redpanda_broker/Makefile: -------------------------------------------------------------------------------- 1 | PYTHON ?= python 2 | PIP ?= pip 3 | 4 | DIR := tests 5 | SCRIPTS := $(wildcard $(DIR)/*_test.py) 6 | 7 | # Target to process each file 8 | .PHONY: all 9 | all: $(FILES) 10 | 11 | .PHONY: do 12 | do: mocks test 13 | 14 | .PHONY: mocks 15 | mocks: 16 | mkdir -p /app/library && \ 17 | cp tests/mocks/* /app/library 18 | 19 | .PHONY: test 20 | test: $(SCRIPTS) 21 | @echo "running $(DIR) $(SCRIPTS)" 22 | @EXIT_CODE=0; \ 23 | for FILE in $(SCRIPTS); do \ 24 | echo "Running $$FILE"; \ 25 | $(PYTHON) $$FILE || EXIT_CODE=1; \ 26 | done; \ 27 | exit $$EXIT_CODE 28 | 29 | .PHONY: up 30 | up: 31 | docker build -t redpanda-broker-test . && docker run redpanda-broker-test 32 | 33 | .PHONY: compose-up 34 | compose-up: 35 | docker compose up --build 36 | -------------------------------------------------------------------------------- /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | agents: 2 | queue: "k8s-builders" 3 | 4 | steps: 5 | - label: "broker template tests" 6 | plugins: 7 | - docker-compose#v5.2.0: 8 | config: roles/redpanda_broker/docker-compose.yml 9 | run: testctr 10 | - label: "connect template tests" 11 | plugins: 12 | - docker-compose#v5.2.0: 13 | config: roles/redpanda_connect/docker-compose.yml 14 | run: testctr 15 | - label: "console template tests" 16 | plugins: 17 | - docker-compose#v5.2.0: 18 | config: roles/redpanda_console/docker-compose.yml 19 | run: testctr 20 | - label: "logging template tests" 21 | plugins: 22 | - docker-compose#v5.2.0: 23 | config: roles/redpanda_logging/docker-compose.yml 24 | run: testctr 25 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/fips-assert.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Check OS FIPS mode with /usr/bin/fips-mode-setup 3 | ansible.builtin.command: 4 | cmd: /usr/bin/fips-mode-setup --is-enabled 5 | register: os_fips_mode 6 | when: 7 | - ansible_os_family == "RedHat" 8 | changed_when: false 9 | failed_when: 10 | - os_fips_mode.rc not in fips_check_exit_codes 11 | 12 | - name: Ensure FIPS not disabled" 13 | ansible.builtin.assert: 14 | that: 15 | - os_fips_mode.rc == 2 and fips_mode != "enabled" 16 | fail_msg: "OS does not have FIPS correctly enabled (fips-mode-setup exit code == {{os_fips_mode.rc}}) and Redpanda Ansible 'fips_mode' setting is possibly enabled (currently set to {{fips_mode}}); this will cause Redpanda startup failures. set fips_mode=permissive for unsafe bypass" 17 | 18 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/copy-truststore.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set facts 3 | ansible.builtin.set_fact: 4 | truststore_file: tls/certs/{{ansible_hostname}}/{{ truststore_file_name}} 5 | delegate_to: localhost 6 | tags: copy_truststore 7 | 8 | - name: Ensure /etc/redpanda/truststores exists 9 | ansible.builtin.file: 10 | path: "{{ redpanda_truststores_dir }}" 11 | state: directory 12 | owner: redpanda 13 | group: redpanda 14 | mode: "0755" 15 | tags: copy_truststore 16 | 17 | - name: Copy truststore to remote hosts in connect group 18 | ansible.builtin.copy: 19 | src: "{{ truststore_file }}" 20 | dest: "{{ redpanda_truststores_dir }}/{{ truststore_file_name }}" 21 | owner: redpanda 22 | group: redpanda 23 | mode: "0644" 24 | tags: copy_truststore 25 | -------------------------------------------------------------------------------- /roles/redpanda_console/tasks/install-console-rpm.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install redpanda console (rpm) 3 | ansible.builtin.package: 4 | name: 5 | - redpanda-console{{ redpanda_version_suffix }} 6 | state: "{{ redpanda_install_status }}" 7 | update_cache: true 8 | register: console_install_result 9 | 10 | # Get installed version if 'latest' was requested 11 | - name: Get installed console version 12 | ansible.builtin.shell: rpm -q --qf '%{VERSION}' redpanda-console 13 | register: console_version_result 14 | changed_when: false 15 | when: redpanda_version == 'latest' 16 | 17 | - name: Update template choice based on installed version 18 | ansible.builtin.set_fact: 19 | use_pre_v3_template: "{{ console_version_result.stdout is version('3.0.0', '<') }}" 20 | when: redpanda_version == 'latest' 21 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/safe-restart.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart Redpanda Connect 3 | ansible.builtin.systemd_service: 4 | name: redpanda-connect 5 | state: restarted 6 | when: 7 | - restart_required | bool 8 | - inventory_hostname == cluster_host 9 | 10 | - name: Wait for Kafka Connect REST API to be healthy 11 | ansible.builtin.uri: 12 | url: "{{ 'https' if connect_tls_enabled else 'http' }}://localhost:{{ rest_port }}/connectors" 13 | method: GET 14 | status_code: 200 15 | validate_certs: false 16 | register: health_check_result 17 | until: health_check_result.status == 200 18 | retries: "{{ connect_restart_health_check_retries }}" 19 | delay: "{{ connect_restart_health_check_delay }}" 20 | when: 21 | - restart_required | bool 22 | - inventory_hostname == cluster_host 23 | - ansible_play_hosts | length > 1 24 | ignore_errors: false 25 | -------------------------------------------------------------------------------- /roles/binary_bundler/tasks/deb_bundle.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set Redpanda DEB 3 | ansible.builtin.set_fact: 4 | deb_links_actual: "{{ is_using_unstable | bool | ternary(deb_unstable_download_urls, deb_download_urls) }}" 5 | tags: 6 | - airgap-tarball-install 7 | 8 | - name: Download Redpanda DEBs to local machine 9 | ansible.builtin.get_url: 10 | url: "{{ item.value }}" 11 | dest: "{{ download_directory }}" 12 | mode: '0644' 13 | delegate_to: localhost 14 | tags: 15 | - airgap-tarball-install 16 | loop: "{{ deb_links_actual | dict2items }}" 17 | 18 | - name: Create tarball of Redpanda DEBs 19 | community.general.archive: 20 | path: "{{ download_directory }}/redpanda*.deb" 21 | dest: "{{ download_directory }}/redpanda_debs.tar.gz" 22 | mode: '0644' 23 | format: gz 24 | remove: true 25 | delegate_to: localhost 26 | tags: 27 | - airgap-tarball-install 28 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/generate-log4j-config.yml: -------------------------------------------------------------------------------- 1 | - name: Use user-provided logging config content if defined 2 | ansible.builtin.copy: 3 | content: "{{ log4j_config_override_content }}" 4 | dest: "{{ redpanda_connect_config_dir }}/connect-log4j.properties" 5 | owner: "{{ redpanda_user }}" 6 | group: "{{ redpanda_group }}" 7 | mode: '0644' 8 | force: true 9 | when: log4j_config_override_content is defined 10 | register: log4j_user_config_result 11 | 12 | - name: Use default logging config if user-provided content is not defined 13 | ansible.builtin.template: 14 | src: log4j.properties.j2 15 | dest: "{{ redpanda_connect_config_dir }}/connect-log4j.properties" 16 | owner: "{{ redpanda_user }}" 17 | group: "{{ redpanda_group }}" 18 | mode: '0644' 19 | force: true 20 | when: log4j_config_override_content is not defined 21 | register: log4j_default_config_result 22 | -------------------------------------------------------------------------------- /roles/redpanda_connect/templates/connect-distributed/connect-distributed.properties.j2: -------------------------------------------------------------------------------- 1 | bootstrap.servers={% for host in advertised_ips %}{{ host }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %} 2 | 3 | schema.registry.url={{ schema_registry_actual }} 4 | group.id={{ group_id }} 5 | config.storage.topic={{ config_storage_topic }} 6 | offset.storage.topic={{ offset_storage_topic }} 7 | status.storage.topic={{ status_storage_topic }} 8 | rest.port={{ rest_port }} 9 | rest.advertised.host.name={{ rest_advertised_host_name_actual }} 10 | rest.advertised.port={{ rest_advertised_port }} 11 | plugin.path={{ plugin_path }} 12 | plugin.discovery={{ plugin_discovery }} 13 | key.converter={{ key_converter }} 14 | value.converter={{ value_converter }} 15 | 16 | {% if not connect_tls_enabled %} 17 | security.protocol=PLAINTEXT 18 | producer.security.protocol=PLAINTEXT 19 | consumer.security.protocol=PLAINTEXT 20 | admin.security.protocol=PLAINTEXT 21 | {% endif %} 22 | -------------------------------------------------------------------------------- /roles/redpanda_logging/templates/redpanda-logrotate.conf.j2: -------------------------------------------------------------------------------- 1 | {{ redpanda_logging_log_file }} { 2 | {{ redpanda_logging_logrotate_frequency }} 3 | rotate {{ redpanda_logging_logrotate_rotate }} 4 | maxsize {{ redpanda_logging_logrotate_maxsize }} 5 | su {{ redpanda_logging_owner }} {{ redpanda_logging_group }} 6 | {% if redpanda_logging_logrotate_compress %} 7 | compress 8 | {% endif %} 9 | {% if redpanda_logging_logrotate_delaycompress %} 10 | delaycompress 11 | {% endif %} 12 | {% if redpanda_logging_logrotate_notifempty %} 13 | notifempty 14 | {% endif %} 15 | {% if redpanda_logging_logrotate_create %} 16 | create {{ redpanda_logging_file_mode }} {{ redpanda_logging_owner }} {{ redpanda_logging_group }} 17 | {% endif %} 18 | {% if redpanda_logging_logrotate_sharedscripts %} 19 | sharedscripts 20 | {% endif %} 21 | postrotate 22 | {{ redpanda_logging_logrotate_postrotate_command | indent(8) }} 23 | endscript 24 | } -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/install-nightly-build-rpm.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set distribution-specific variables 3 | set_fact: 4 | distro: "{{ ansible_distribution | lower }}" 5 | codename: "{{ ansible_distribution_version }}" 6 | arch: "{{ ansible_architecture }}" 7 | 8 | - name: Install yum-utils 9 | dnf: 10 | name: yum-utils 11 | state: present 12 | 13 | - name: Import Cloudsmith GPG key 14 | rpm_key: 15 | key: "{{ cloudsmith_gpg_key_url }}" 16 | state: present 17 | 18 | - name: Download repository configuration 19 | get_url: 20 | url: "{{ cloudsmith_config_url_rpm }}?distro={{ distro }}&codename={{ codename }}" 21 | dest: /tmp/redpanda-redpanda-nightly.repo 22 | mode: '0644' 23 | become: true 24 | 25 | - name: Add repository configuration 26 | command: dnf config-manager --add-repo '/tmp/redpanda-redpanda-nightly.repo' 27 | become: true 28 | 29 | - name: Make DNF cache 30 | command: dnf -q makecache -y --disablerepo='*' --enablerepo='redpanda-redpanda-nightly' --enablerepo='redpanda-redpanda-nightly-source' 31 | become: true 32 | -------------------------------------------------------------------------------- /roles/redpanda_connect/README.md: -------------------------------------------------------------------------------- 1 | # Redpanda Connect Role 2 | 3 | Installs Redpanda Connect. Currently in a limited alpha release. 4 | 5 | ## Overriding Config Files 6 | 7 | There's a few different ways we're doing this as the files themselves are very different in structure and purpose. 8 | 9 | For the connect-distributed file we're providing two default files that can be merged together for TLS. Then we allow the user to merge in whatever they want through an environment variable. 10 | 11 | For the log4j and logging.properties file we allow straight content replace and only straight content replace as these are text files that do not need per-host modification. 12 | 13 | For the jmx exporter file we're able to offer more complex merge and replace functionality as it is json and json dict merging is well supported in ansible. 14 | 15 | For the systemd unit we allow passing in additional options to the kafka system but nothing much else 16 | 17 | For the environment file named java-home we're allowing the user to provide additional values that can be appended after the existing ones. 18 | -------------------------------------------------------------------------------- /roles/system_setup/tasks/install-node-deps-deb.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set custom fact based on ansible_system_vendor 3 | ansible.builtin.set_fact: 4 | using_gcp: true 5 | when: ansible_system_vendor == 'Google' 6 | 7 | - name: Node dependencies - Set APT proxy 8 | ansible.builtin.template: 9 | src: apt_proxy.j2 10 | dest: /etc/apt/apt.conf.d/proxy.conf 11 | mode: '0644' 12 | become: true 13 | when: 14 | - https_proxy_value is defined and https_proxy_value | length > 0 15 | - (create_pkg_mgr_proxy | default(false)) is true 16 | - (using_gcp | default(false)) is false 17 | 18 | - name: Node dependencies - update packages - debian 19 | tags: 20 | - node_deps 21 | ansible.builtin.apt: 22 | update_cache: true 23 | force_apt_get: true 24 | environment: 25 | https_proxy: "{{ https_proxy_value | default('') }}" 26 | 27 | - name: Node dependencies - Install Debian prerequisites 28 | tags: 29 | - node_deps 30 | ansible.builtin.package: 31 | name: "{{ debian_prerequisite_packages }}" 32 | state: present 33 | update_cache: true 34 | environment: 35 | https_proxy: "{{ https_proxy_value | default('') }}" 36 | -------------------------------------------------------------------------------- /roles/client_config/tasks/install-rpk.yml: -------------------------------------------------------------------------------- 1 | - name: Set architecture fact for url 2 | ansible.builtin.set_fact: 3 | rpk_arch: "{{ 'amd64' if ansible_architecture == 'x86_64' else 'arm64' if ansible_architecture == 'aarch64' else 'unsupported' }}" 4 | 5 | - name: Ensure unzip is installed 6 | ansible.builtin.package: 7 | name: unzip 8 | state: present 9 | become: true 10 | 11 | - name: Download Redpanda rpk zip for amd64 12 | ansible.builtin.get_url: 13 | url: "{{ rpk_url | default('https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-' + rpk_arch + '.zip') }}" 14 | dest: "/tmp/rpk-linux-{{ rpk_arch }}.zip" 15 | mode: '0644' 16 | when: rpk_arch in ['amd64', 'arm64'] 17 | 18 | - name: Unzip Redpanda rpk to /usr/local/bin 19 | ansible.builtin.unarchive: 20 | src: "/tmp/rpk-linux-{{ rpk_arch }}.zip" 21 | dest: /usr/local/bin 22 | remote_src: true 23 | become: true 24 | when: rpk_arch in ['amd64', 'arm64'] 25 | 26 | - name: Fail for unsupported architecture 27 | ansible.builtin.fail: 28 | msg: "The architecture '{{ ansible_architecture }}' is not supported for Redpanda rpk installation" 29 | when: rpk_arch == 'unsupported' 30 | -------------------------------------------------------------------------------- /roles/demo_certs/tasks/generate-truststore.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set facts 3 | ansible.builtin.set_fact: 4 | node_cert: tls/certs/{{ansible_hostname}}/node.crt 5 | truststore_file: tls/certs/{{ansible_hostname}}/{{ truststore_file_name }} 6 | delegate_to: localhost 7 | tags: generate_truststore 8 | 9 | - name: Ensure the truststore file is absent 10 | delegate_to: localhost 11 | ansible.builtin.file: 12 | path: "{{ truststore_file }}" 13 | state: absent 14 | tags: generate_truststore 15 | 16 | - name: Create the truststore 17 | delegate_to: localhost 18 | ansible.builtin.command: 19 | cmd: | 20 | keytool -importcert -noprompt -file {{ node_cert }} -alias {{ ansible_hostname }} -keystore {{ truststore_file }} -storepass {{ truststore_password }} -storetype {{ truststore_type }} 21 | tags: generate_truststore 22 | 23 | - name: Import the CA certificate into the truststore 24 | delegate_to: localhost 25 | ansible.builtin.command: 26 | cmd: | 27 | keytool -importcert -noprompt -file {{ root_ca_dir }}/ca.crt -alias ca -keystore {{ truststore_file }} -storepass {{ truststore_password }} -storetype {{ truststore_type }} 28 | tags: generate_truststore 29 | -------------------------------------------------------------------------------- /roles/redpanda_broker/templates/configs/tiered_storage.j2: -------------------------------------------------------------------------------- 1 | { 2 | "cluster": { 3 | "cloud_storage_bucket": "{{ tiered_storage_bucket_name if tiered_storage_bucket_name is defined }}", 4 | "cloud_storage_enable_remote_read": "{{ cloud_storage_enable_remote_read }}", 5 | "cloud_storage_enable_remote_write": "{{ cloud_storage_enable_remote_write }}", 6 | "cloud_storage_region": "{{ cloud_storage_region if cloud_storage_region is defined }}", 7 | "cloud_storage_credentials_source": "{{ cloud_storage_credentials_source }}", 8 | # cloud_storage_enabled is true if the bucket name is set and the cloud_storage_credentials_source isn't set to "config_file". 9 | # If config_file is required, you'll need to specify the cloud_storage_access_key, cloud_storage_secret_key and also cloud_storage_enabled=true 10 | # using the redpanda variable which gets merged into these templates. 11 | "cloud_storage_enabled": "{{ true if tiered_storage_bucket_name is defined and tiered_storage_bucket_name|d('')|length > 0 and cloud_storage_credentials_source != "config_file" else false }}"{% if cloud_storage_credentials_source == 'gcp_instance_metadata' %}, 12 | "cloud_storage_api_endpoint": "storage.googleapis.com", 13 | {% endif %} 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /roles/redpanda_connect/templates/redpanda-connect.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Redpanda's distribution of Apache Kafka Connect in distributed mode 3 | Documentation=https://docs.redpanda.com/ 4 | After=network.target 5 | 6 | [Service] 7 | Type=simple 8 | User={{ redpanda_user }} 9 | Group={{ redpanda_group }} 10 | TimeoutStartSec={{ timeout_start_sec }} 11 | Environment="CONNECT_PLUGIN_PATH={{ redpanda_connect_home }}/plugins" 12 | {% if kafka_log4j_opts is defined and kafka_log4j_opts != "" %} 13 | Environment="KAFKA_LOG4J_OPTS={{ kafka_log4j_opts }}" 14 | {% endif %} 15 | {% if kafka_opts is defined and kafka_opts != "" %} 16 | Environment="KAFKA_OPTS={{ kafka_opts }}{% if jmx_ssl is defined and jmx_ssl != false -%} \ 17 | -Djavax.net.ssl.keyStore={{ jmx_keystore_path }} \ 18 | -Djavax.net.ssl.keyStorePassword={{ jmx_keystore_password }} \ 19 | -Djavax.net.ssl.trustStore={{ jmx_truststore_path }} \ 20 | -Djavax.net.ssl.trustStorePassword={{ jmx_truststore_password }}{% endif -%} 21 | {%- if kafka_additional_opts is defined and kafka_additional_opts != "" %} {{ kafka_additional_opts }}{% endif %}" 22 | {% endif %} 23 | 24 | ExecStart={{ redpanda_connect_home }}/bin/connect-distributed.sh {{ redpanda_connect_config_dir }}/{{ connect_distributed_config_file }} 25 | TimeoutStopSec=180 26 | Restart=no 27 | 28 | [Install] 29 | WantedBy=multi-user.target 30 | -------------------------------------------------------------------------------- /roles/redpanda_console/tasks/install-console-deb.yml: -------------------------------------------------------------------------------- 1 | - name: Install redpanda console (deb) 2 | ansible.builtin.package: 3 | name: 4 | - redpanda-console{{ redpanda_version_suffix }} 5 | state: "{{ redpanda_install_status }}" 6 | update_cache: true 7 | environment: 8 | https_proxy: "{{ https_proxy_value | default('') }}" 9 | http_proxy: "{{ https_proxy_value | default('') }}" 10 | register: console_install_result 11 | when: https_proxy_value is defined and https_proxy_value | length > 0 12 | 13 | - name: Install redpanda console (deb) 14 | ansible.builtin.package: 15 | name: 16 | - redpanda-console{{ redpanda_version_suffix }} 17 | state: "{{ redpanda_install_status }}" 18 | update_cache: true 19 | register: console_install_result 20 | when: https_proxy_value is not defined or https_proxy_value | length == 0 21 | 22 | # Get installed version if 'latest' was requested 23 | - name: Get installed console version 24 | ansible.builtin.shell: dpkg-query -W -f='${Version}' redpanda-console 25 | register: console_version_result 26 | changed_when: false 27 | when: redpanda_version == 'latest' 28 | 29 | - name: Update template choice based on installed version 30 | ansible.builtin.set_fact: 31 | use_pre_v3_template: "{{ console_version_result.stdout is version('3.0.0', '<') }}" 32 | when: redpanda_version == 'latest' 33 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/install-nightly-build-deb.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set distribution-specific variables 3 | set_fact: 4 | distro: "{{ ansible_distribution | lower }}" 5 | codename: "{{ ansible_distribution_release }}" 6 | arch: "{{ ansible_architecture }}" 7 | 8 | - name: Update apt cache 9 | apt: 10 | update_cache: yes 11 | 12 | - name: Install required packages 13 | apt: 14 | name: 15 | - debian-keyring 16 | - debian-archive-keyring 17 | - apt-transport-https 18 | - gnupg 19 | state: present 20 | 21 | - name: Download Cloudsmith GPG key 22 | get_url: 23 | url: "{{ cloudsmith_gpg_key_url }}" 24 | dest: /tmp/cloudsmith_gpg.key 25 | mode: '0644' 26 | 27 | - name: Dearmor and add Cloudsmith GPG key 28 | ansible.builtin.shell: gpg --dearmor < /tmp/cloudsmith_gpg.key > {{ keyring_location }} 29 | args: 30 | creates: "{{ keyring_location }}" 31 | become: true 32 | 33 | - name: Set correct permissions for the keyring file 34 | file: 35 | path: "{{ keyring_location }}" 36 | mode: '0644' 37 | become: true 38 | 39 | - name: Add Cloudsmith repository configuration 40 | get_url: 41 | url: "{{ cloudsmith_config_url_deb }}?distro={{ distro }}&codename={{ codename }}&arch={{ arch }}&component=main" 42 | dest: /etc/apt/sources.list.d/redpanda-redpanda-nightly.list 43 | mode: '0644' 44 | become: true 45 | -------------------------------------------------------------------------------- /roles/system_setup/tasks/install-node-deps-rpm.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set custom fact based on ansible_system_vendor 3 | ansible.builtin.set_fact: 4 | using_gcp: true 5 | when: ansible_system_vendor == 'Google' 6 | 7 | - name: Node dependencies - Set DNF proxy 8 | ansible.builtin.template: 9 | src: dnf_proxy.j2 10 | dest: /etc/dnf/dnf.conf 11 | mode: '0644' 12 | become: true 13 | when: 14 | - rpm_proxy is defined and rpm_proxy | length > 0 15 | - (create_pkg_mgr_proxy | default(false)) is true 16 | - (using_gcp | default(false)) is false 17 | 18 | - name: Node dependencies - RPM setup 19 | tags: 20 | - node_deps 21 | become: true 22 | block: 23 | - name: Node dependencies - delete unused repos 24 | ansible.builtin.file: 25 | name: "{{item}}" 26 | state: absent 27 | with_items: 28 | - /etc/yum.repos.d/fedora-updates-modular.repo 29 | - /etc/yum.repos.d/fedora-cisco-openh264.repo 30 | - name: Node dependencies - install base deps 31 | ansible.builtin.package: 32 | name: 33 | - iotop 34 | - mdadm 35 | - xfsprogs 36 | state: present 37 | use: dnf 38 | 39 | - name: Node dependencies - Install RPM packages 40 | tags: 41 | - node_deps 42 | become: true 43 | ansible.builtin.package: 44 | name: "{{ rpm_prerequisite_packages }}" 45 | allowerasing: true 46 | state: present 47 | update_cache: true 48 | -------------------------------------------------------------------------------- /roles/demo_certs/tasks/create-ca.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Creates a local CA in $dir/ca (if one does not exist) 3 | # 4 | 5 | --- 6 | - name: Check existence of Root CA directory {{ root_ca_dir }} 7 | delegate_to: localhost 8 | ansible.builtin.file: 9 | path: "{{ root_ca_dir }}" 10 | state: directory 11 | mode: "0700" 12 | tags: 13 | - create_ca 14 | 15 | - name: Copy the ca.conf file to CA 16 | delegate_to: localhost 17 | ansible.builtin.template: 18 | src: ca.conf.tpl 19 | dest: "{{ root_ca_dir }}/ca.conf" 20 | mode: "0644" 21 | tags: 22 | - create_ca 23 | 24 | - name: Generate an OpenSSH keypair for the CA 25 | delegate_to: localhost 26 | ansible.builtin.command: 27 | chdir: "{{ root_ca_dir }}" 28 | cmd: openssl genrsa -out {{ root_ca_dir }}/ca.key 2048 29 | creates: "{{ root_ca_dir }}/ca.key" 30 | tags: 31 | - create_ca 32 | 33 | - name: Generate self-signed CA certificate 34 | delegate_to: localhost 35 | ansible.builtin.command: 36 | chdir: "{{ root_ca_dir }}" 37 | cmd: openssl req -new -x509 -config ca.conf -key ca.key -out ca.crt -days 365 -batch 38 | creates: "{{ root_ca_dir }}/ca.crt" 39 | tags: 40 | - create_ca 41 | 42 | - name: Initialize CA data files 43 | delegate_to: localhost 44 | ansible.builtin.shell: | 45 | cd {{ root_ca_dir }} 46 | rm -f index.txt serial.txt 47 | touch index.txt 48 | echo '01' > serial.txt 49 | changed_when: false 50 | tags: 51 | - create_ca 52 | -------------------------------------------------------------------------------- /roles/demo_certs/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # creates a local CA for demo purposes, should not be used in prod 3 | - name: Create local ca 4 | ansible.builtin.include_tasks: create-ca.yml 5 | run_once: true 6 | when: 7 | - create_demo_certs | default(false) | bool 8 | 9 | # Generates CSRs from the demo local CA 10 | - name: Generate cert signing requests 11 | ansible.builtin.include_tasks: generate-csrs.yml 12 | when: 13 | - create_demo_certs | default(false) | bool 14 | 15 | # Issues certs from the demo local CA 16 | - name: Issue certs 17 | ansible.builtin.include_tasks: issue-certs.yml 18 | when: 19 | - create_demo_certs | default(false) | bool 20 | 21 | # Generates a Java truststore containing the CA certificate and the node certificate on localhost 22 | - name: Generate CSRs for keystores 23 | ansible.builtin.include_tasks: generate-csrs-keystore.yml 24 | when: 25 | - create_keystore | default(false) | bool 26 | 27 | # Generates a Java truststore containing the CA certificate and the node certificate on localhost 28 | - name: Generate certs for keystores 29 | ansible.builtin.include_tasks: issue-certs-keystore.yml 30 | when: 31 | - create_keystore | default(false) | bool 32 | - 33 | # Generates a Java truststore containing the CA certificate and the node certificate on localhost 34 | - name: Generate truststore on localhost for connect group 35 | ansible.builtin.include_tasks: generate-truststore.yml 36 | when: 37 | - create_keystore | default(false) | bool 38 | -------------------------------------------------------------------------------- /roles/redpanda_broker/library/mock_ansible_module.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | from ansible.module_utils.basic import AnsibleModule 4 | 5 | def main(): 6 | module = AnsibleModule( 7 | argument_spec=dict( 8 | mock_type=dict(type='str', required=True, choices=['shell', 'template']), 9 | cmd=dict(type='str', required=False), 10 | mock_stdout=dict(type='str', required=False), 11 | mock_rc=dict(type='int', required=False), 12 | src=dict(type='str', required=False), 13 | dest=dict(type='str', required=False), 14 | owner=dict(type='str', required=False), 15 | group=dict(type='str', required=False), 16 | mode=dict(type='str', required=False), 17 | mock_changed=dict(type='bool', required=False) 18 | ) 19 | ) 20 | 21 | mock_type = module.params['mock_type'] 22 | 23 | if mock_type == 'shell': 24 | mock_stdout = module.params['mock_stdout'] 25 | mock_rc = module.params['mock_rc'] 26 | 27 | if mock_rc == 0: 28 | module.exit_json(changed=False, stdout=mock_stdout, stderr='', rc=mock_rc) 29 | else: 30 | module.fail_json(msg="Command failed", stdout=mock_stdout, stderr='Command failed', rc=mock_rc) 31 | 32 | elif mock_type == 'template': 33 | mock_changed = module.params['mock_changed'] 34 | module.exit_json(changed=mock_changed, dest=module.params['dest']) 35 | 36 | 37 | if __name__ == '__main__': 38 | main() 39 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tests/mocks/mock_ansible_module.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | from ansible.module_utils.basic import AnsibleModule 4 | 5 | def main(): 6 | module = AnsibleModule( 7 | argument_spec=dict( 8 | mock_type=dict(type='str', required=True, choices=['shell', 'template']), 9 | cmd=dict(type='str', required=False), 10 | mock_stdout=dict(type='str', required=False), 11 | mock_rc=dict(type='int', required=False), 12 | src=dict(type='str', required=False), 13 | dest=dict(type='str', required=False), 14 | owner=dict(type='str', required=False), 15 | group=dict(type='str', required=False), 16 | mode=dict(type='str', required=False), 17 | mock_changed=dict(type='bool', required=False) 18 | ) 19 | ) 20 | 21 | mock_type = module.params['mock_type'] 22 | 23 | if mock_type == 'shell': 24 | mock_stdout = module.params['mock_stdout'] 25 | mock_rc = module.params['mock_rc'] 26 | 27 | if mock_rc == 0: 28 | module.exit_json(changed=False, stdout=mock_stdout, stderr='', rc=mock_rc) 29 | else: 30 | module.fail_json(msg="Command failed", stdout=mock_stdout, stderr='Command failed', rc=mock_rc) 31 | 32 | elif mock_type == 'template': 33 | mock_changed = module.params['mock_changed'] 34 | module.exit_json(changed=mock_changed, dest=module.params['dest']) 35 | 36 | 37 | if __name__ == '__main__': 38 | main() 39 | -------------------------------------------------------------------------------- /roles/demo_certs/templates/ca.conf.tpl: -------------------------------------------------------------------------------- 1 | # OpenSSL CA configuration file 2 | [ ca ] 3 | default_ca = local_ca 4 | 5 | [ local_ca ] 6 | dir = {{ root_ca_dir }} 7 | database = $dir/index.txt 8 | serial = $dir/serial.txt 9 | default_days = 730 10 | default_md = sha256 11 | copy_extensions = copy 12 | unique_subject = no 13 | 14 | # Used to create the CA certificate. 15 | [ req ] 16 | prompt = no 17 | distinguished_name = distinguished_name 18 | x509_extensions = extensions 19 | 20 | [ root_ca_distinguished_name ] 21 | commonName = Test TLS CA 22 | stateOrProvinceName = NY 23 | countryName = US 24 | emailAddress = hi@vectorized.io 25 | organizationName = Vectorized 26 | organizationalUnitName = Vectorized Test 27 | 28 | [ distinguished_name ] 29 | organizationName = Vectorized 30 | commonName = Vectorized Test CA 31 | 32 | [ extensions ] 33 | keyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment,keyCertSign 34 | basicConstraints = critical,CA:true,pathlen:1 35 | 36 | # Common policy for nodes and users. 37 | [ signing_policy ] 38 | organizationName = supplied 39 | commonName = optional 40 | 41 | # Used to sign node certificates. 42 | [ signing_node_req ] 43 | keyUsage = critical,digitalSignature,keyEncipherment 44 | extendedKeyUsage = serverAuth,clientAuth 45 | 46 | # Used to sign client certificates. 47 | [ signing_client_req ] 48 | keyUsage = critical,digitalSignature,keyEncipherment 49 | extendedKeyUsage = clientAuth 50 | -------------------------------------------------------------------------------- /roles/redpanda_connect/templates/connect-distributed/connect-distributed-tls.properties.j2: -------------------------------------------------------------------------------- 1 | security.protocol=SSL 2 | ssl.protocol={{ ssl_protocol }} 3 | ssl.debug=true 4 | listeners=https://0.0.0.0:8083 5 | rest.advertised.listeners=https://{{ inventory_hostname }}:8083 6 | ssl.truststore.location={{ssl_truststore_location}} 7 | ssl.truststore.password={{ssl_truststore_password}} 8 | ssl.truststore.type={{ssl_truststore_type}} 9 | producer.ssl.truststore.location={{producer_ssl_truststore_location}} 10 | producer.ssl.truststore.password={{producer_ssl_truststore_password}} 11 | consumer.ssl.truststore.location={{consumer_ssl_truststore_location}} 12 | consumer.ssl.truststore.password={{consumer_ssl_truststore_password}} 13 | admin.ssl.truststore.location={{admin_ssl_truststore_location}} 14 | admin.ssl.truststore.password={{admin_ssl_truststore_password}} 15 | ssl.keystore.location={{ssl_keystore_location}} 16 | ssl.keystore.password={{ssl_keystore_password}} 17 | ssl.keystore.type={{ssl_keystore_type}} 18 | producer.ssl.keystore.location={{producer_ssl_keystore_location}} 19 | producer.ssl.keystore.password={{producer_ssl_keystore_password}} 20 | producer.ssl.keystore.type={{producer_ssl_keystore_type}} 21 | consumer.ssl.keystore.location={{consumer_ssl_keystore_location}} 22 | consumer.ssl.keystore.password={{consumer_ssl_keystore_password}} 23 | consumer.ssl.keystore.type={{consumer_ssl_keystore_type}} 24 | admin.ssl.keystore.location={{admin_ssl_keystore_location}} 25 | admin.ssl.keystore.password={{admin_ssl_keystore_password}} 26 | admin.ssl.keystore.type={{admin_ssl_keystore_type}} 27 | -------------------------------------------------------------------------------- /meta/runtime.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Collections must specify a minimum required ansible version to upload 3 | # to galaxy 4 | requires_ansible: '>=2.16.4' 5 | 6 | # Content that Ansible needs to load from another location or that has 7 | # been deprecated/removed 8 | # plugin_routing: 9 | # action: 10 | # redirected_plugin_name: 11 | # redirect: ns.col.new_location 12 | # deprecated_plugin_name: 13 | # deprecation: 14 | # removal_version: "4.0.0" 15 | # warning_text: | 16 | # See the porting guide on how to update your playbook to 17 | # use ns.col.another_plugin instead. 18 | # removed_plugin_name: 19 | # tombstone: 20 | # removal_version: "2.0.0" 21 | # warning_text: | 22 | # See the porting guide on how to update your playbook to 23 | # use ns.col.another_plugin instead. 24 | # become: 25 | # cache: 26 | # callback: 27 | # cliconf: 28 | # connection: 29 | # doc_fragments: 30 | # filter: 31 | # httpapi: 32 | # inventory: 33 | # lookup: 34 | # module_utils: 35 | # modules: 36 | # netconf: 37 | # shell: 38 | # strategy: 39 | # terminal: 40 | # test: 41 | # vars: 42 | 43 | # Python import statements that Ansible needs to load from another location 44 | # import_redirection: 45 | # ansible_collections.ns.col.plugins.module_utils.old_location: 46 | # redirect: ansible_collections.ns.col.plugins.module_utils.new_location 47 | 48 | # Groups of actions/modules that take a common set of options 49 | # action_groups: 50 | # group_name: 51 | # - module1 52 | # - module2 53 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/safe-restart.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # attempt to do a rolling, serial restart by putting each node into maintenance 3 | # mode, restarting, and putting it back into service. This is throttled to allow 4 | # only one node out of service at a time. 5 | # 6 | - name: Enable Maintenance Mode 7 | ansible.builtin.command: 8 | cmd: rpk cluster maintenance enable {{ node_id }} --wait {{ redpanda_rpk_opts }} 9 | no_log: "{{ redpanda_broker_no_log }}" 10 | when: 11 | - restart_required 12 | - inventory_hostname == cluster_host 13 | - ansible_play_hosts | length > 1 14 | 15 | # We need to (actually) generate the node config once the node is in MM, otherwise the `rpk cluster maintenance` command 16 | # will fail 17 | - name: Generate Node config - post-bootstrap runs 18 | ansible.builtin.template: 19 | src: redpanda.yml 20 | dest: /etc/redpanda/redpanda.yaml 21 | owner: redpanda 22 | group: redpanda 23 | mode: "0644" 24 | register: nodeconfig_result 25 | when: 26 | - is_initialized 27 | - inventory_hostname == cluster_host 28 | 29 | - name: Restart Redpanda 30 | ansible.builtin.systemd: 31 | name: redpanda 32 | state: restarted 33 | when: 34 | - restart_required 35 | - inventory_hostname == cluster_host 36 | 37 | - name: Disable Maintenance Mode 38 | ansible.builtin.command: 39 | cmd: rpk cluster maintenance disable {{ node_id }} {{ redpanda_rpk_opts }} 40 | no_log: "{{ redpanda_broker_no_log }}" 41 | when: 42 | - restart_required 43 | - inventory_hostname == cluster_host 44 | - ansible_play_hosts | length > 1 45 | -------------------------------------------------------------------------------- /roles/binary_bundler/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure basearch is defined 3 | ansible.builtin.assert: 4 | that: 5 | - basearch != '' 6 | fail_msg: "Variable 'basearch' must be defined! This is equivalent to basearch (which can be gotten with uname -m) on the target system" 7 | 8 | - name: Ensure redpanda_version is defined 9 | ansible.builtin.assert: 10 | that: 11 | - redpanda_version != '' 12 | fail_msg: "Variable 'redpanda_version' must be defined! Don't use latest here, it won't work" 13 | 14 | - name: Ensure os_distribution is defined 15 | ansible.builtin.assert: 16 | that: 17 | - os_distribution != '' 18 | fail_msg: "Variable 'os_distribution' must be defined! This is equivalent to ansible_distribution (can be gotten with . /etc/os-release && echo $ID ) on the target system" 19 | 20 | - name: Ensure os_distribution_major_version is defined 21 | ansible.builtin.assert: 22 | that: 23 | - os_distribution_major_version != '' 24 | fail_msg: "Variable 'os_distribution_major_version' must be defined! This is equivalent to ansible_distribution_major_version (can be gotten with . /etc/os-release && echo ${VERSION_ID%%.*} ) on the target system" 25 | 26 | - name: Ensure rpm_or_deb is defined 27 | ansible.builtin.assert: 28 | that: 29 | - rpm_or_deb != '' 30 | fail_msg: "Variable 'rpm_or_deb' must be defined!" 31 | 32 | - name: Bundle RPM 33 | ansible.builtin.include_tasks: rpm_bundle.yml 34 | when: rpm_or_deb == "rpm" 35 | 36 | - name: Bundle DEB 37 | ansible.builtin.include_tasks: deb_bundle.yml 38 | when: rpm_or_deb == "deb" 39 | -------------------------------------------------------------------------------- /roles/redpanda_console/tasks/configure-deb-repository.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set Redpanda GPG Key and Repository details 3 | ansible.builtin.set_fact: 4 | rp_key_deb_actual: "{{ rp_key_deb }}" 5 | rp_key_path_deb_actual: "{{ rp_key_path_deb }}" 6 | rp_repo_signing_deb_actual: "{{ rp_repo_signing_deb }}" 7 | rp_repo_signing_src_deb_actual: "{{ rp_repo_signing_src_deb }}" 8 | 9 | - name: Download and import Redpanda GPG Key with proxy 10 | become: true 11 | ansible.builtin.shell: | 12 | curl -x "{{ https_proxy_value }}" -1sLf "{{ rp_key_deb_actual }}" | gpg --dearmor > "{{ rp_key_path_deb_actual }}" 13 | args: 14 | creates: "{{ rp_key_path_deb_actual }}" 15 | when: https_proxy_value is defined and https_proxy_value | length > 0 16 | 17 | - name: Download and import Redpanda GPG Key without proxy 18 | become: true 19 | args: 20 | creates: "{{ rp_key_path_deb_actual }}" 21 | ansible.builtin.shell: | 22 | curl -1sLf "{{ rp_key_deb_actual }}" | gpg --dearmor > "{{ rp_key_path_deb_actual }}" 23 | when: https_proxy_value is not defined or https_proxy_value | length == 0 24 | 25 | - name: Add Redpanda DEB repository 26 | ansible.builtin.copy: 27 | dest: "/etc/apt/sources.list.d/redpanda-redpanda.list" 28 | content: "{{ rp_repo_signing_deb_actual }}\n" 29 | owner: root 30 | group: root 31 | mode: '0644' 32 | 33 | - name: Add Redpanda DEB repository (source) 34 | ansible.builtin.copy: 35 | dest: "/etc/apt/sources.list.d/redpanda-redpanda.list" 36 | content: "{{ rp_repo_signing_deb_actual }}\n" 37 | owner: root 38 | group: root 39 | mode: '0644' 40 | -------------------------------------------------------------------------------- /roles/system_setup/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Check if ca_cert_file is defined 3 | ansible.builtin.fail: 4 | msg: "A valid path must be provided for ca_cert_file ex: tls/certs/ca.crt" 5 | when: 6 | - handle_cert_install | default(false) | bool 7 | - ca_cert_file is not defined 8 | 9 | - name: Check if node_cert_file is defined 10 | ansible.builtin.fail: 11 | msg: "A valid path must be provided for node_cert_file ex: tls/certs/node.crt" 12 | when: 13 | - handle_cert_install | default(false) | bool 14 | - node_cert_file is not defined 15 | 16 | # installs necessary dependencies for running redpanda 17 | - name: Install dependencies 18 | ansible.builtin.include_tasks: install-node-deps-deb.yml 19 | when: ansible_facts['os_family']|lower == 'debian' 20 | 21 | # installs necessary dependencies for running redpanda 22 | - name: Install dependencies 23 | ansible.builtin.include_tasks: install-node-deps-rpm.yml 24 | when: ansible_facts['os_family']|lower == 'redhat' 25 | 26 | - name: Create Redpanda User 27 | ansible.builtin.include_tasks: create-redpanda-user.yml 28 | when: 29 | - prep_data_dir | default(true) | bool 30 | 31 | # configures the data dir. won't work if Install dependencies isn't run first 32 | - name: Prep data dir 33 | ansible.builtin.include_tasks: prepare-data-dir.yml 34 | when: 35 | - prep_data_dir | default(true) | bool 36 | 37 | # sets permissions on the data directory -- separate from creating for increased flexibility 38 | - name: Set data dir perms 39 | ansible.builtin.include_tasks: data-dir-perms.yml 40 | when: 41 | - data_dir_perms | default(true) | bool 42 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/copy-keystore.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set facts 3 | ansible.builtin.set_fact: 4 | keystores_file: tls/certs/{{ansible_hostname}}/{{ keystores_file_name}} 5 | delegate_to: localhost 6 | tags: copy_truststore 7 | 8 | - name: Ensure /etc/redpanda/keystores exists 9 | ansible.builtin.file: 10 | path: "{{ redpanda_keystores_dir }}" 11 | state: directory 12 | owner: redpanda 13 | group: redpanda 14 | mode: "0755" 15 | tags: copy_truststore 16 | 17 | - name: Check if keystores file exists 18 | stat: 19 | path: "{{ keystores_file }}" 20 | register: keystores_file_stat 21 | 22 | - name: Copy keystore to remote hosts in connect group 23 | ansible.builtin.copy: 24 | src: "{{ keystores_file }}" 25 | dest: "{{ redpanda_keystores_dir }}/{{ keystores_file_name }}" 26 | owner: redpanda 27 | group: redpanda 28 | mode: "0644" 29 | tags: copy_truststore 30 | when: keystores_file_stat.stat.exists 31 | 32 | - name: Create the keystore with a self-signed certificate 33 | ansible.builtin.command: 34 | cmd: | 35 | openssl {{ ssl_keystore_type | lower }} -export -in {{ redpanda_cert_file }} -inkey {{ redpanda_key_file }} -out {{ ssl_keystore_location }} -name {{ ansible_hostname }} -password pass:{{ ssl_keystore_password }} 36 | tags: generate_keystore 37 | when: not keystores_file_stat.stat.exists 38 | 39 | - name: Ensure /etc/redpanda/keystores exists 40 | ansible.builtin.file: 41 | path: "{{ ssl_keystore_location }}" 42 | state: file 43 | owner: redpanda 44 | group: redpanda 45 | mode: "0755" 46 | tags: copy_truststore 47 | when: not keystores_file_stat.stat.exists 48 | -------------------------------------------------------------------------------- /roles/redpanda_console/tasks/configure-console.yml: -------------------------------------------------------------------------------- 1 | - name: Ensure /etc/redpanda exists 2 | ansible.builtin.file: 3 | path: /etc/redpanda 4 | state: directory 5 | mode: "0755" 6 | 7 | - name: Create list of advertised_ips 8 | ansible.builtin.set_fact: 9 | advertised_ips: "{{ 10 | groups['redpanda'] | map('extract', hostvars) | map(attribute='inventory_hostname' if (advertise_public_ips | default(False) | bool) else 'private_ip') | list 11 | }}" 12 | 13 | - name: Create list of kafka_connect_advertised_ips 14 | ansible.builtin.set_fact: 15 | kafka_connect_advertised_ips: "{{ 16 | groups['connect'] | map('extract', hostvars) | map(attribute='inventory_hostname' if (advertise_public_ips | default(False) | bool) else 'private_ip') | list 17 | }}" 18 | when: "'connect' in groups" 19 | 20 | - name: Reset configuration 21 | ansible.builtin.set_fact: 22 | console_config: { } 23 | 24 | - name: Generate configurations 25 | ansible.builtin.set_fact: 26 | console_config: "{{ console_config | combine(lookup('template', console_config_template.template), recursive=True) }}" 27 | loop: "{{ console_config_templates }}" 28 | loop_control: 29 | loop_var: console_config_template 30 | when: console_config_template.condition | default(True) 31 | 32 | - name: Merge with user-provided overrides (via rpconsole variable) 33 | ansible.builtin.set_fact: 34 | console_config: "{{ console_config | combine((rpconsole | default('{}') | from_json), recursive=True) }}" 35 | 36 | - name: Generate Console config 37 | ansible.builtin.template: 38 | src: console.yml 39 | dest: /etc/redpanda/redpanda-console-config.yaml 40 | owner: redpanda 41 | group: redpanda 42 | mode: "0644" 43 | -------------------------------------------------------------------------------- /roles/redpanda_logging/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Redpanda Logging Configuration 3 | 4 | # Enable logging configuration 5 | redpanda_logging_enabled: true 6 | 7 | redpanda_logging_log_file: /var/log/redpanda.log 8 | 9 | # Log directory ownership (OS-specific defaults) 10 | redpanda_logging_owner: "{{ 'root' if ansible_os_family == 'RedHat' else 'syslog' }}" 11 | redpanda_logging_group: "{{ 'root' if ansible_os_family == 'RedHat' else 'adm' }}" 12 | redpanda_logging_dir_mode: '0755' 13 | redpanda_logging_file_mode: '0640' 14 | 15 | # Rsyslog configuration 16 | redpanda_logging_rsyslog_enabled: true 17 | redpanda_logging_rsyslog_priority: 40 # Priority for rsyslog config file 18 | 19 | redpanda_logging_program: rpk 20 | 21 | # Logrotate configuration 22 | redpanda_logging_logrotate_enabled: true 23 | redpanda_logging_logrotate_frequency: daily 24 | redpanda_logging_logrotate_rotate: 7 # Keep 7 rotated logs 25 | redpanda_logging_logrotate_maxsize: 100M 26 | redpanda_logging_logrotate_compress: true 27 | redpanda_logging_logrotate_delaycompress: true 28 | redpanda_logging_logrotate_notifempty: true 29 | redpanda_logging_logrotate_create: true 30 | redpanda_logging_logrotate_sharedscripts: true 31 | 32 | # Additional logrotate options 33 | redpanda_logging_logrotate_postrotate_command: | 34 | for pidfile in /var/run/rsyslogd.pid /run/rsyslogd.pid; do 35 | if [ -f "$pidfile" ]; then 36 | /bin/kill -HUP `cat "$pidfile" 2> /dev/null` 2> /dev/null || true 37 | break 38 | fi 39 | done 40 | 41 | # Systemd logging configuration 42 | redpanda_logging_systemd_enabled: false # Enable systemd-specific logging config 43 | redpanda_logging_systemd_max_level: info # MaxLevelStore for journald 44 | redpanda_logging_systemd_forward_to_syslog: true 45 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/generate-connect-distributed.yml: -------------------------------------------------------------------------------- 1 | - name: Create list of advertised_ips 2 | ansible.builtin.set_fact: 3 | advertised_ips: "{{ 4 | groups['redpanda'] | map('extract', hostvars) | map(attribute='inventory_hostname' if (advertise_public_address | bool) else 'private_ip') | list 5 | }}" 6 | 7 | - name: Schema Registry Actual 8 | ansible.builtin.set_fact: 9 | schema_registry_actual: "{{ schema_registry_url | default('http' + ('s' if connect_tls_enabled else '') + '://' + groups['redpanda'][0] + ':8081') }}" 10 | 11 | - name: Rest Advertised Host Name Actual 12 | ansible.builtin.set_fact: 13 | rest_advertised_host_name_actual: "{{ rest_advertised_host_name | default(hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}" 14 | - name: Set default configuration 15 | ansible.builtin.set_fact: 16 | connect_distributed_config: "{{ lookup('template', 'connect-distributed/connect-distributed.properties.j2') }}" 17 | 18 | - name: Include TLS configuration if enabled 19 | ansible.builtin.set_fact: 20 | connect_distributed_config: "{{ connect_distributed_config + lookup('template', 'connect-distributed/connect-distributed-tls.properties.j2') }}" 21 | when: connect_tls_enabled 22 | 23 | - name: Merge user-provided configuration 24 | ansible.builtin.set_fact: 25 | connect_distributed_config: "{{ connect_distributed_config + '\n' + connect_distributed_user_config }}" 26 | when: connect_distributed_user_config is defined 27 | 28 | - name: Instantiate config file 29 | ansible.builtin.copy: 30 | content: "{{ connect_distributed_config }}" 31 | dest: "{{ redpanda_connect_config_dir }}/connect-distributed.properties" 32 | owner: "{{ redpanda_user }}" 33 | group: "{{ redpanda_group }}" 34 | mode: '0644' 35 | force: true 36 | register: connect_distributed_config_result 37 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/install-certs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create redpanda group 3 | ansible.builtin.group: 4 | name: "{{ redpanda_group }}" 5 | state: present 6 | system: true 7 | become: true 8 | tags: 9 | - install_certs 10 | 11 | 12 | - name: Install certs - create redpanda user if it doesn't exist already 13 | tags: 14 | - install_certs 15 | ansible.builtin.user: 16 | name: "{{ redpanda_user }}" 17 | group: "{{ redpanda_group }}" 18 | system: true 19 | shell: /usr/sbin/nologin 20 | create_home: false 21 | state: present 22 | become: true 23 | 24 | - name: Install certs - ensure /etc/redpanda/certs exists 25 | tags: 26 | - install_certs 27 | ansible.builtin.file: 28 | path: "{{ redpanda_certs_dir }}" 29 | state: directory 30 | owner: redpanda 31 | group: redpanda 32 | mode: "0755" 33 | 34 | - name: Install_certs - Copy CA Certs 35 | tags: 36 | - install_certs 37 | ansible.builtin.copy: 38 | src: "{{ ca_cert_file }}" 39 | dest: "{{ redpanda_truststore_file }}" 40 | owner: redpanda 41 | group: redpanda 42 | force: "{{ overwrite_certs | default('no') | bool }}" 43 | mode: "0644" 44 | 45 | - name: Install_certs - Copy Node Certs 46 | tags: 47 | - install_certs 48 | ansible.builtin.copy: 49 | src: "{{ node_cert_file }}" 50 | dest: "{{ redpanda_cert_file }}" 51 | owner: redpanda 52 | group: redpanda 53 | force: "{{ overwrite_certs | default('no') | bool }}" 54 | mode: "0644" 55 | 56 | - name: Install_certs - Copy Node Keys 57 | tags: 58 | - install_certs 59 | ansible.builtin.copy: 60 | src: "{{ node_key_file }}" 61 | dest: "{{ redpanda_key_file }}" 62 | owner: redpanda 63 | group: redpanda 64 | force: "{{ overwrite_certs | default('no') | bool }}" 65 | mode: "0644" 66 | when: 67 | - not (node_key_file | default('') == '') 68 | -------------------------------------------------------------------------------- /galaxy.yml: -------------------------------------------------------------------------------- 1 | ### REQUIRED 2 | # The namespace of the collection. This can be a company/brand/organization or product namespace under which all 3 | # content lives. May only contain alphanumeric lowercase characters and underscores. Namespaces cannot start with 4 | # underscores or numbers and cannot contain consecutive underscores 5 | namespace: redpanda 6 | 7 | # The name of the collection. Has the same character restrictions as 'namespace' 8 | name: cluster 9 | 10 | # The version of the collection. Must be compatible with semantic versioning 11 | version: 0.10.0 12 | 13 | # The path to the Markdown (.md) readme file. This path is relative to the root of the collection 14 | readme: README.md 15 | 16 | # A list of the collection's content authors. Can be just the name or in the format 'Full Name (url) 17 | # @nicks:irc/im.site#channel' 18 | authors: 19 | - Devex Team devex@redpanda.com 20 | 21 | ### OPTIONAL but strongly recommended 22 | # A short summary description of the collection 23 | description: Redpanda Ansible Collection 24 | 25 | # The path to the license file for the collection. This path is relative to the root of the collection. This key is 26 | # mutually exclusive with 'license' 27 | license_file: LICENSE 28 | 29 | # A list of tags you want to associate with the collection for indexing/searching. A tag name has the same character 30 | # requirements as 'namespace' and 'name' 31 | tags: ["redpanda", "redpanda_data", "data", "redpanda_cluster", "kafka", "apache_kafka"] 32 | 33 | # The URL of the originating SCM repository 34 | repository: https://github.com/redpanda-data/deployment-automation 35 | 36 | # The URL to any online docs 37 | documentation: https://docs.redpanda.com/docs/home 38 | 39 | # The URL to the homepage of the collection/project 40 | homepage: https://redpanda.com/ 41 | 42 | # The URL to the collection issue tracker 43 | issues: https://github.com/redpanda-data/deployment-automation/issues 44 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/install-certs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create redpanda group 3 | ansible.builtin.group: 4 | name: "{{ redpanda_group }}" 5 | state: present 6 | system: true 7 | become: true 8 | tags: 9 | - install_certs 10 | 11 | - name: Create redpanda user if it doesn't exist already 12 | tags: 13 | - install_certs 14 | ansible.builtin.user: 15 | name: "{{ redpanda_user }}" 16 | group: "{{ redpanda_group }}" 17 | system: true 18 | shell: /usr/sbin/nologin 19 | create_home: false 20 | state: present 21 | become: true 22 | 23 | - name: Ensure /etc/redpanda/certs exists 24 | tags: 25 | - install_certs 26 | ansible.builtin.file: 27 | path: "{{ redpanda_certs_dir }}" 28 | state: directory 29 | owner: redpanda 30 | group: redpanda 31 | mode: "0755" 32 | 33 | - name: Copy CA Cert 34 | tags: 35 | - install_certs 36 | ansible.builtin.copy: 37 | src: "{{ ca_cert_file }}" 38 | dest: "{{ redpanda_truststore_file }}" 39 | owner: redpanda 40 | group: redpanda 41 | force: "{{ overwrite_certs | default('no') | bool }}" 42 | mode: "0644" 43 | 44 | - name: Copy Node Certs 45 | tags: 46 | - install_certs 47 | ansible.builtin.copy: 48 | src: "{{ node_cert_file }}" 49 | dest: "{{ redpanda_cert_file }}" 50 | owner: redpanda 51 | group: redpanda 52 | force: "{{ overwrite_certs | default('no') | bool }}" 53 | mode: "0644" 54 | 55 | - name: Copy Node Keys 56 | tags: 57 | - install_certs 58 | ansible.builtin.copy: 59 | src: "{{ node_key_file }}" 60 | dest: "{{ redpanda_key_file }}" 61 | owner: redpanda 62 | group: redpanda 63 | force: "{{ overwrite_certs | default('no') | bool }}" 64 | mode: "0644" 65 | when: 66 | - not (node_key_file | default('') == '') 67 | - handle_cert_install | default(false) | bool 68 | - not (create_demo_certs | default(false) | bool) 69 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/configure-deb-repository.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set Redpanda GPG Key and Repository details 3 | ansible.builtin.set_fact: 4 | rp_key_deb_actual: "{{ is_using_unstable | bool | default(false) | ternary(rp_key_deb_unstable, rp_key_deb) }}" 5 | rp_key_path_deb_actual: "{{ is_using_unstable | bool | default(false) | ternary(rp_key_path_deb_unstable, rp_key_path_deb) }}" 6 | rp_repo_signing_deb_actual: "{{ is_using_unstable | bool | default(false) | ternary(rp_repo_signing_deb_unstable, rp_repo_signing_deb) }}" 7 | rp_repo_signing_src_deb_actual: "{{ is_using_unstable | bool | default(false) | ternary(rp_repo_signing_src_deb_unstable, rp_repo_signing_src_deb) }}" 8 | 9 | - name: Download and import Redpanda GPG Key with proxy 10 | become: true 11 | ansible.builtin.shell: | 12 | curl -x "{{ https_proxy_value }}" -1sLf "{{ rp_key_deb_actual }}" | gpg --dearmor > "{{ rp_key_path_deb_actual }}" 13 | args: 14 | creates: "{{ rp_key_path_deb_actual }}" 15 | when: https_proxy_value is defined and https_proxy_value | length > 0 16 | 17 | - name: Download and import Redpanda GPG Key without proxy 18 | become: true 19 | args: 20 | creates: "{{ rp_key_path_deb_actual }}" 21 | ansible.builtin.shell: | 22 | curl -1sLf "{{ rp_key_deb_actual }}" | gpg --dearmor > "{{ rp_key_path_deb_actual }}" 23 | when: https_proxy_value is not defined or https_proxy_value | length == 0 24 | 25 | - name: Add Redpanda DEB repository 26 | ansible.builtin.copy: 27 | dest: "/etc/apt/sources.list.d/redpanda-redpanda.list" 28 | content: "{{ rp_repo_signing_deb_actual }}\n" 29 | owner: root 30 | group: root 31 | mode: '0644' 32 | 33 | - name: Add Redpanda DEB repository (source) 34 | ansible.builtin.copy: 35 | dest: "/etc/apt/sources.list.d/redpanda-redpanda.list" 36 | content: "{{ rp_repo_signing_deb_actual }}\n" 37 | owner: root 38 | group: root 39 | mode: '0644' 40 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tests/restart_required.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | gather_facts: false 4 | vars: 5 | is_initialized: false 6 | nodeconfig_result: 7 | changed: false 8 | package_result: 9 | results: [] 10 | restart_node: true 11 | mock_shell_stdout: "false" 12 | mock_shell_rc: 0 13 | redpanda_rpk_opts: "test" 14 | node_id: "0" 15 | no_log: false 16 | 17 | tasks: 18 | - name: Set is_initialized fact based on controller_stat 19 | ansible.builtin.set_fact: 20 | is_initialized: "{{ controller_stat.stat.isdir is defined and controller_stat.stat.isdir }}" 21 | - name: Generate Node config - post-bootstrap runs 22 | mock_ansible_module: 23 | mock_type: template 24 | src: redpanda.yml 25 | dest: /etc/redpanda/redpanda.yaml 26 | owner: redpanda 27 | group: redpanda 28 | mode: "0644" 29 | mock_changed: "{{ mock_template_changed }}" 30 | check_mode: true 31 | register: nodeconfig_result 32 | when: is_initialized 33 | - name: Check if restart needed 34 | mock_ansible_module: 35 | mock_type: shell 36 | cmd: "rpk cluster config status {{ redpanda_rpk_opts }} | grep '^{{ node_id }} ' | awk '{ print $3 }' | grep -E 'true|false'" 37 | mock_stdout: "{{ mock_shell_stdout }}" 38 | mock_rc: "{{ mock_shell_rc }}" 39 | register: restart_required_rc 40 | changed_when: false 41 | no_log: "{{ no_log }}" 42 | - name: Establish whether restart required 43 | ansible.builtin.set_fact: 44 | restart_required: '{{ ("true" in restart_required_rc.stdout or (is_initialized and (nodeconfig_result.changed or package_result.results is defined and ("Removed" in package_result.results or "1 upgraded" in package_result.results)))) and (restart_node | default("true") | bool) }}' 45 | - name: Debug restart_required 46 | ansible.builtin.debug: 47 | var: restart_required 48 | -------------------------------------------------------------------------------- /roles/binary_bundler/tasks/rpm_bundle.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set Redpanda RPM 3 | ansible.builtin.set_fact: 4 | # resolves stable vs unstable 5 | rp_standard_rpm_actuals: "{{ is_using_unstable | bool | ternary(rpm_standard_unstable_download_urls, rpm_standard_download_urls) }}" 6 | rp_noarch_rpm_actuals: "{{ is_using_unstable | bool | ternary(rpm_noarch_unstable_download_urls, rpm_noarch_download_urls) }}" 7 | rp_source_rpm_actuals: "{{ is_using_unstable | bool | ternary(rpm_source_unstable_download_urls, rpm_source_download_urls) }}" 8 | tags: 9 | - airgap-tarball-install 10 | 11 | - name: Download Redpanda RPMs (standard) to local machine 12 | ansible.builtin.get_url: 13 | url: "{{ item.value }}" 14 | dest: "{{ download_directory }}/{{ item.key }}__standard.rpm" 15 | mode: '0644' 16 | loop: "{{ rp_standard_rpm_actuals | dict2items }}" 17 | delegate_to: localhost 18 | tags: 19 | - airgap-tarball-install 20 | 21 | - name: Download Redpanda RPMs (noarch) to local machine 22 | ansible.builtin.get_url: 23 | url: "{{ item.value }}" 24 | dest: "{{ download_directory }}/{{ item.key }}__noarch.rpm" 25 | mode: '0644' 26 | loop: "{{ rp_noarch_rpm_actuals | dict2items }}" 27 | delegate_to: localhost 28 | tags: 29 | - airgap-tarball-install 30 | 31 | - name: Download Redpanda RPMs (source) to local machine 32 | ansible.builtin.get_url: 33 | url: "{{ item.value }}" 34 | dest: "{{ download_directory }}/{{ item.key }}__source.rpm" 35 | mode: '0644' 36 | loop: "{{ rp_source_rpm_actuals | dict2items }}" 37 | delegate_to: localhost 38 | tags: 39 | - airgap-tarball-install 40 | 41 | - name: Create tarball of Redpanda RPMs 42 | community.general.archive: 43 | path: "{{ download_directory }}/redpanda*__*.rpm" 44 | dest: "{{ download_directory }}/redpanda_rpms.tar.gz" 45 | format: gz 46 | remove: true 47 | mode: '0644' 48 | delegate_to: localhost 49 | tags: 50 | - airgap-tarball-install 51 | -------------------------------------------------------------------------------- /roles/redpanda_console/tasks/install-certs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create redpanda group 3 | ansible.builtin.group: 4 | name: "{{ redpanda_group }}" 5 | state: present 6 | system: true 7 | become: true 8 | tags: 9 | - install_certs 10 | 11 | - name: Install certs - create redpanda user if it doesn't exist already 12 | tags: 13 | - install_certs 14 | ansible.builtin.user: 15 | name: "{{ redpanda_user }}" 16 | group: "{{ redpanda_group }}" 17 | system: true 18 | shell: /usr/sbin/nologin 19 | create_home: false 20 | state: present 21 | become: true 22 | 23 | - name: Install certs - ensure /etc/redpanda/certs exists 24 | tags: 25 | - install_certs 26 | ansible.builtin.file: 27 | path: "{{ redpanda_certs_dir }}" 28 | state: directory 29 | owner: redpanda 30 | group: redpanda 31 | mode: "0755" 32 | 33 | - name: Install_certs - Copy CA Certs 34 | tags: 35 | - install_certs 36 | ansible.builtin.copy: 37 | src: "{{ ca_cert_file }}" 38 | dest: "{{ redpanda_truststore_file }}" 39 | owner: redpanda 40 | group: redpanda 41 | force: "{{ overwrite_certs | default('no') | bool }}" 42 | mode: "0644" 43 | 44 | - name: Install_certs - Copy Node Certs 45 | tags: 46 | - install_certs 47 | ansible.builtin.copy: 48 | src: "{{ node_cert_file }}" 49 | dest: "{{ redpanda_cert_file }}" 50 | owner: redpanda 51 | group: redpanda 52 | force: "{{ overwrite_certs | default('no') | bool }}" 53 | mode: "0644" 54 | 55 | - name: Install_certs - Copy Node Keys 56 | tags: 57 | - install_certs 58 | ansible.builtin.copy: 59 | src: "{{ node_key_file }}" 60 | dest: "{{ redpanda_key_file }}" 61 | owner: redpanda 62 | group: redpanda 63 | force: "{{ overwrite_certs | default('no') | bool }}" 64 | mode: "0644" 65 | when: 66 | - not (node_key_file | default('') == '') 67 | - handle_cert_install | default(false) | bool 68 | - not (create_demo_certs | default(false) | bool) 69 | -------------------------------------------------------------------------------- /roles/demo_certs/tasks/issue-certs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set facts 3 | tags: 4 | - issue_certs 5 | ansible.builtin.set_fact: 6 | node_cert: tls/certs/{{ansible_hostname}}/node.crt 7 | node_csr: tls/certs/{{ansible_hostname}}/node.csr 8 | 9 | - name: Check for csr 10 | tags: 11 | - issue_certs 12 | ansible.builtin.file: 13 | path: "{{ node_csr }}" 14 | state: file 15 | mode: "0644" 16 | register: csr_stat 17 | failed_when: csr_stat.state == 'absent' 18 | delegate_to: 127.0.0.1 19 | 20 | - name: Check for cert 21 | tags: 22 | - issue_certs 23 | ansible.builtin.file: 24 | path: "{{ node_cert }}" 25 | state: "{{ 'absent' if (overwrite_certs | default('no') | bool) else 'file' }}" 26 | mode: "0644" 27 | failed_when: false 28 | register: cert_stat 29 | delegate_to: 127.0.0.1 30 | 31 | - name: Check csr modulus 32 | tags: 33 | - issue_certs 34 | ansible.builtin.command: 35 | cmd: openssl req -noout -modulus -in {{ node_csr }} 36 | register: csr_modulus 37 | when: cert_stat.state == 'file' 38 | changed_when: false 39 | delegate_to: 127.0.0.1 40 | 41 | - name: Check cert modulus 42 | tags: 43 | - issue_certs 44 | ansible.builtin.command: 45 | cmd: openssl x509 -noout -modulus -in {{ node_cert }} 46 | register: cert_modulus 47 | when: cert_stat.state == 'file' 48 | failed_when: cert_modulus.stdout != csr_modulus.stdout 49 | changed_when: false 50 | delegate_to: 127.0.0.1 51 | 52 | - name: Issue Certs 53 | tags: 54 | - issue_certs 55 | ansible.builtin.command: 56 | cmd: | 57 | openssl ca 58 | -config tls/ca/ca.conf 59 | -keyfile tls/ca/ca.key 60 | -cert tls/ca/ca.crt 61 | -policy signing_policy 62 | -extensions signing_node_req 63 | -in {{ node_csr }} 64 | -out {{ node_cert }} 65 | -outdir tls/certs/ 66 | -batch 67 | creates: tls/certs/{{ansible_hostname}}/node.crt 68 | delegate_to: 127.0.0.1 69 | register: result_issue_certs 70 | until: result_issue_certs.rc == 0 71 | retries: 3 72 | delay: 10 73 | -------------------------------------------------------------------------------- /roles/redpanda_broker/templates/configs/tls.j2: -------------------------------------------------------------------------------- 1 | { 2 | "node": { 3 | "redpanda": { 4 | "admin_api_tls": { 5 | "enabled": true, 6 | "require_client_auth": {{ require_client_auth }}, 7 | "key_file": "{{ redpanda_key_file }}", 8 | "cert_file": "{{ redpanda_cert_file }}", 9 | "truststore_file": "{{ redpanda_truststore_file }}" 10 | }, 11 | "kafka_api_tls": { 12 | "enabled": true, 13 | "require_client_auth": {{ require_client_auth }}, 14 | "key_file": "{{ redpanda_key_file }}", 15 | "cert_file": "{{ redpanda_cert_file }}", 16 | "truststore_file": "{{ redpanda_truststore_file }}" 17 | }, 18 | "rpc_server_tls": { 19 | "enabled": true, 20 | "require_client_auth": {{ require_client_auth }}, 21 | "key_file": "{{ redpanda_key_file }}", 22 | "cert_file": "{{ redpanda_cert_file }}", 23 | "truststore_file": "{{ redpanda_truststore_file }}" 24 | } 25 | }, 26 | "rpk": { 27 | "admin_api": { 28 | "tls": { 29 | "truststore_file": "{{ redpanda_truststore_file }}" 30 | } 31 | }, 32 | "kafka_api": { 33 | "tls": { 34 | "truststore_file": "{{ redpanda_truststore_file }}" 35 | } 36 | } 37 | }, 38 | "pandaproxy":{ 39 | "pandaproxy_api_tls": [ 40 | { 41 | "enabled": true, 42 | "require_client_auth": {{ require_client_auth }}, 43 | "key_file": "{{ redpanda_key_file }}", 44 | "cert_file": "{{ redpanda_cert_file }}", 45 | "truststore_file": "{{ redpanda_truststore_file }}" 46 | } 47 | ] 48 | }, 49 | "schema_registry":{ 50 | "schema_registry_api_tls": [ 51 | { 52 | "enabled": true, 53 | "require_client_auth": {{ require_client_auth }}, 54 | "key_file": "{{ redpanda_key_file }}", 55 | "cert_file": "{{ redpanda_cert_file }}", 56 | "truststore_file": "{{ redpanda_truststore_file }}" 57 | } 58 | ] 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /roles/demo_certs/tasks/issue-certs-keystore.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set facts 3 | tags: 4 | - issue_certs 5 | ansible.builtin.set_fact: 6 | node_cert: tls/certs/{{ansible_hostname}}/node.crt 7 | node_csr: tls/certs/{{ansible_hostname}}/node.csr 8 | 9 | - name: Check for csr 10 | tags: 11 | - issue_certs 12 | ansible.builtin.file: 13 | path: "{{ node_csr }}" 14 | state: file 15 | mode: "0644" 16 | register: csr_stat 17 | failed_when: csr_stat.state == 'absent' 18 | delegate_to: 127.0.0.1 19 | 20 | - name: Check for cert 21 | tags: 22 | - issue_certs 23 | ansible.builtin.file: 24 | path: "{{ node_cert }}" 25 | state: "{{ 'absent' if (overwrite_certs | default('no') | bool) else 'file' }}" 26 | mode: "0644" 27 | failed_when: false 28 | register: cert_stat 29 | delegate_to: 127.0.0.1 30 | 31 | - name: Check csr modulus 32 | tags: 33 | - issue_certs 34 | ansible.builtin.command: 35 | cmd: openssl req -noout -modulus -in {{ node_csr }} 36 | register: csr_modulus 37 | when: cert_stat.state == 'file' 38 | changed_when: false 39 | delegate_to: 127.0.0.1 40 | 41 | - name: Check cert modulus 42 | tags: 43 | - issue_certs 44 | ansible.builtin.command: 45 | cmd: openssl x509 -noout -modulus -in {{ node_cert }} 46 | register: cert_modulus 47 | when: cert_stat.state == 'file' 48 | failed_when: cert_modulus.stdout != csr_modulus.stdout 49 | changed_when: false 50 | delegate_to: 127.0.0.1 51 | 52 | - name: Issue Certs 53 | tags: 54 | - issue_certs 55 | ansible.builtin.command: 56 | cmd: | 57 | openssl ca 58 | -config tls/ca/ca.conf 59 | -keyfile tls/ca/ca.key 60 | -cert tls/ca/ca.crt 61 | -policy signing_policy 62 | -extensions signing_node_req 63 | -in {{ node_csr }} 64 | -out {{ node_cert }} 65 | -outdir tls/certs/ 66 | -batch 67 | creates: tls/certs/{{ansible_hostname}}/node.crt 68 | delegate_to: 127.0.0.1 69 | register: result_issue_certs 70 | until: result_issue_certs.rc == 0 71 | retries: 3 72 | delay: 10 73 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/install-rp-deb.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set custom fact based on ansible_system_vendor 3 | ansible.builtin.set_fact: 4 | using_gcp: true 5 | when: ansible_system_vendor == 'Google' 6 | 7 | - name: Set fact for package names 8 | ansible.builtin.set_fact: 9 | redpanda_version_suffix: "{{ '' if redpanda_version == 'latest' else ('=' if ansible_os_family == 'Debian' else '-') + redpanda_version }}" 10 | needs_split_packages: >- 11 | {{ redpanda_version == 'latest' or 12 | (redpanda_version is version('24.2.0', '>=')) or 13 | development_build | default(false) | bool }} 14 | 15 | - name: Install redpanda (pre 24.2) 16 | ansible.builtin.apt: 17 | name: "redpanda{{ redpanda_version_suffix }}" 18 | state: "{{ redpanda_install_status }}" 19 | update_cache: true 20 | allow_unauthenticated: "{{ using_gcp | default(false) }}" 21 | allow_downgrade: "{{ allow_downgrade | default(false) }}" 22 | install_recommends: "{{ install_recommends | default(false) }}" 23 | force: yes 24 | environment: 25 | https_proxy: "{{ https_proxy_value | default('') }}" 26 | http_proxy: "{{ https_proxy_value | default('') }}" 27 | register: package_result 28 | when: not needs_split_packages 29 | 30 | - name: Install redpanda (post 24.2) 31 | ansible.builtin.apt: 32 | name: 33 | - "redpanda{{ redpanda_version_suffix }}" 34 | - "redpanda-rpk{{ redpanda_version_suffix }}" 35 | - "redpanda-tuner{{ redpanda_version_suffix }}" 36 | state: "{{ redpanda_install_status }}" 37 | update_cache: true 38 | allow_unauthenticated: "{{ using_gcp | default(false) }}" 39 | allow_downgrade: "{{ allow_downgrade | default(false) }}" 40 | install_recommends: "{{ install_recommends | default(false) }}" 41 | force: yes 42 | environment: 43 | https_proxy: "{{ https_proxy_value | default('') }}" 44 | http_proxy: "{{ https_proxy_value | default('') }}" 45 | register: package_result 46 | when: needs_split_packages 47 | 48 | - name: Set data dir file perms 49 | ansible.builtin.file: 50 | path: "{{ redpanda_data_directory }}" 51 | owner: redpanda 52 | group: redpanda 53 | -------------------------------------------------------------------------------- /roles/redpanda_console/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure redpanda_version is defined 3 | ansible.builtin.assert: 4 | that: 5 | - redpanda_version != '' 6 | fail_msg: "Variable 'redpanda_version' must be defined!" 7 | when: not (install_certs_only | default(false) | bool) 8 | 9 | # Set fact for package version suffix 10 | - name: Set fact for package version suffix 11 | ansible.builtin.set_fact: 12 | redpanda_version_suffix: "{{ '' if redpanda_version == 'latest' else ('=' if ansible_os_family == 'Debian' else '-') + redpanda_version }}" 13 | use_pre_v3_template: "{{ redpanda_version != 'latest' and redpanda_version is version('3.0.0', '<') }}" 14 | 15 | # Copies certs set in ca_cert_file and node_cert_file into the redpanda nodes. see defaults for the default values 16 | # only necessary when TLS is enabled 17 | - name: Install certs 18 | ansible.builtin.include_tasks: install-certs.yml 19 | when: 20 | - handle_cert_install | default(false) | bool 21 | 22 | # End play if install_certs_only is true 23 | - name: End play if only installing certs 24 | ansible.builtin.meta: end_play 25 | when: install_certs_only | default(false) | bool 26 | 27 | - name: Create user and group 28 | ansible.builtin.include_tasks: create-user.yml 29 | 30 | - name: Configure Redpanda RPM Repository 31 | ansible.builtin.include_tasks: configure-rpm-repository.yml 32 | when: 33 | - ansible_os_family == 'RedHat' 34 | - enable_airgap is false 35 | 36 | - name: Configure Redpanda DEB Repository 37 | ansible.builtin.include_tasks: configure-deb-repository.yml 38 | when: 39 | - ansible_os_family == 'Debian' 40 | - enable_airgap is false 41 | 42 | - name: Install Redpanda Console 43 | ansible.builtin.include_tasks: install-console-deb.yml 44 | when: 45 | - ansible_os_family == 'Debian' 46 | 47 | - name: Configure Redpanda Console 48 | ansible.builtin.include_tasks: configure-console.yml 49 | 50 | - name: Install Redpanda Console 51 | ansible.builtin.include_tasks: install-console-rpm.yml 52 | when: 53 | - ansible_os_family == 'RedHat' 54 | 55 | - name: Start Redpanda Console 56 | ansible.builtin.include_tasks: start-console.yml 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ansible Collection for Redpanda 2 | 3 | Redpanda Ansible Collection that enables provisioning and managing a [Redpanda](https://www.redpanda.com/) cluster. 4 | 5 | ## Usage 6 | 7 | ### Required Steps: Deploying Redpanda 8 | 9 | More information on consuming this collection is [available here](https://docs.redpanda.com/docs/deploy/deployment-option/self-hosted/manual/production/production-deployment-automation/) in our official documentation. 10 | 11 | You can also see some example playbooks in our [deployment-automation](https://github.com/redpanda-data/deployment-automation/tree/main/ansible) repository. 12 | 13 | ### Creating and Publishing the Collection 14 | 15 | To build the collection you first need to bump the version number listed in ansible/redpanda/cluster/galaxy.yml 16 | 17 | You will probably 18 | need [appropriate permissions](https://galaxy.ansible.com/docs/contributing/namespaces.html#adding-administrators-to-a-namespace) 19 | on the namespace for this to work. 20 | 21 | Once that's all sorted, run the following shell script with 22 | an [API key from Ansible Galaxy](https://galaxy.ansible.com/me/preferences) 23 | 24 | 25 | ```shell 26 | ansible-galaxy collection build 27 | ansible-galaxy collection publish redpanda-cluster-*.tar.gz --token -s https://galaxy.ansible.com/api/ 28 | ``` 29 | 30 | ## Troubleshooting 31 | 32 | ### On Mac OS X, Python unable to fork workers 33 | 34 | If you see something like this: 35 | 36 | ``` 37 | ok: [34.209.26.177] => {“changed”: false, “stat”: {“exists”: false}} 38 | objc[57889]: +[__NSCFConstantString initialize] may have been in progress in another thread when fork() was called. 39 | objc[57889]: +[__NSCFConstantString initialize] may have been in progress in another thread when fork() was called. We cannot safely call it or ignore it in the fork() child process. Crashing instead. Set a breakpoint on objc_initializeAfterForkError to debug. 40 | ERROR! A worker was found in a dead state 41 | ``` 42 | 43 | You might try resolving by setting an environment variable: 44 | `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` 45 | 46 | See: https://stackoverflow.com/questions/50168647/multiprocessing-causes-python-to-crash-and-gives-an-error-may-have-been-in-progr 47 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/install-rp-rpm.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set fact for package names 3 | ansible.builtin.set_fact: 4 | redpanda_version_suffix: "{{ '' if redpanda_version == 'latest' else '-' + redpanda_version }}" 5 | needs_split_packages: >- 6 | {{ redpanda_version == 'latest' or 7 | (redpanda_version is version('24.2.0', '>=')) or 8 | development_build | default(false) | bool }} 9 | allow_lower_version: "{{ development_build | default(false) | bool }}" 10 | 11 | - name: check OS FIPS mode state to see if configured and safe for Redpanda to proceed with fips_mode=enabled 12 | ansible.builtin.include_tasks: fips-assert.yml 13 | when: 14 | - enable_fips | default(false) | bool 15 | 16 | - name: generate package list to install 17 | ansible.builtin.set_fact: 18 | redpanda_package_list: "{{ ['redpanda-rpk-fips', 'redpanda', 'redpanda-fips', 'redpanda-tuner'] if enable_fips | bool else ['redpanda-rpk','redpanda','redpanda-tuner'] }}" 19 | 20 | - name: generate versioned package list to install 21 | ansible.builtin.set_fact: 22 | redpanda_package_list_versioned: "{{ redpanda_package_list | product([redpanda_version_suffix]) | map('join') | list }}" 23 | 24 | - name: Install redpanda (pre 24.2) 25 | dnf: 26 | name: "redpanda{{ redpanda_version_suffix }}" 27 | state: "{{ redpanda_install_status }}" 28 | update_cache: true 29 | allow_downgrade: "{{ allow_lower_version }}" 30 | environment: 31 | https_proxy: "{{ https_proxy_value | default('') }}" 32 | http_proxy: "{{ https_proxy_value | default('') }}" 33 | register: package_result 34 | when: not needs_split_packages 35 | 36 | 37 | - name: Install redpanda (post 24.2 or development) 38 | dnf: 39 | name: "{{item}}" 40 | state: "{{ redpanda_install_status }}" 41 | update_cache: true 42 | allow_downgrade: "{{ allow_lower_version }}" 43 | environment: 44 | https_proxy: "{{ https_proxy_value | default('') }}" 45 | http_proxy: "{{ https_proxy_value | default('') }}" 46 | register: package_result 47 | when: needs_split_packages 48 | loop: "{{ redpanda_package_list }}" 49 | 50 | - name: Set data dir file perms 51 | ansible.builtin.file: 52 | path: "{{ redpanda_data_directory }}" 53 | owner: redpanda 54 | group: redpanda 55 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/generate-jmx-exporter-config.yml: -------------------------------------------------------------------------------- 1 | # example usage in a playbook 2 | #logging_config_override_content: | 3 | # log4j.rootLogger=INFO, stdout, syslogAppender 4 | # 5 | # log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | # log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 7 | # 8 | # log4j.appender.syslogAppender=org.apache.log4j.net.SyslogAppender 9 | # log4j.appender.syslogAppender.syslogHost=localhost 10 | # log4j.appender.syslogAppender.facility=LOCAL0 11 | # log4j.appender.syslogAppender.layout=org.apache.log4j.PatternLayout 12 | # 13 | # connect.log.pattern=[%d] %p %X{connector.context}%m (%c:%L)%n 14 | # 15 | # log4j.appender.stdout.layout.ConversionPattern=${connect.log.pattern} 16 | # log4j.appender.syslogAppender.layout.ConversionPattern=${connect.log.pattern} 17 | # 18 | # log4j.logger.org.reflections=ERROR 19 | 20 | - name: Use user-provided logging config content if defined 21 | ansible.builtin.copy: 22 | content: "{{ logging_config_override_content }}" 23 | dest: "{{ redpanda_connect_config_dir }}/connect-log4j.properties" 24 | owner: "{{ redpanda_user }}" 25 | group: "{{ redpanda_group }}" 26 | mode: '0644' 27 | force: true 28 | when: logging_config_override_content is defined 29 | 30 | - name: Use default logging config if user-provided content is not defined 31 | ansible.builtin.template: 32 | src: log4j.properties.j2 33 | dest: "{{ redpanda_connect_config_dir }}/connect-log4j.properties" 34 | owner: "{{ redpanda_user }}" 35 | group: "{{ redpanda_group }}" 36 | mode: '0644' 37 | force: true 38 | when: logging_config_override_content is not defined 39 | 40 | - name: Load template contents 41 | ansible.builtin.set_fact: 42 | jmx_exporter_config: "{{ lookup('template', 'jmx-exporter-config.json.j2') }}" 43 | 44 | - name: Merge with user-provided overrides 45 | ansible.builtin.set_fact: 46 | jmx_exporter_config: "{{ jmx_exporter_config | combine((jmx_exporter_override | default('{}') | from_json), recursive=True) }}" 47 | 48 | - name: Instantiate jmx-exporter file 49 | ansible.builtin.copy: 50 | content: "{{ jmx_exporter_config | to_nice_json }}" 51 | dest: "{{ redpanda_connect_config_dir }}/jmx-exporter-config.json" 52 | owner: "{{ redpanda_user }}" 53 | group: "{{ redpanda_group }}" 54 | mode: '0644' 55 | force: true 56 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tests/jmx-exporter-config_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import yaml 4 | from jinja2 import Environment, FileSystemLoader 5 | 6 | class TestJmxExporterConfig(unittest.TestCase): 7 | def test_bootstrap_server_template(self): 8 | # Get the absolute path of the current file 9 | current_dir = os.path.dirname(os.path.abspath(__file__)) 10 | 11 | # Construct the path to the templates directory 12 | templates_dir = os.path.join(current_dir, '..', 'templates') 13 | 14 | # Construct the path to the defaults/main.yml file 15 | defaults_file = os.path.join(current_dir, '..', 'defaults', 'main.yml') 16 | 17 | # Load the defaults from the YAML file 18 | with open(defaults_file, 'r') as f: 19 | defaults = yaml.safe_load(f) 20 | 21 | # Create a Jinja2 environment and load the template from file 22 | env = Environment(loader=FileSystemLoader(templates_dir)) 23 | template = env.get_template('jmx-exporter-config.json.j2') 24 | 25 | # Define the hostvars and groups for rendering the template 26 | hostvars = { 27 | '35.91.106.231': { 28 | 'ansible_host': '35.91.106.231', 29 | }, 30 | '35.88.129.205': { 31 | 'ansible_host': '35.88.129.205', 32 | }, 33 | '54.190.184.126': { 34 | 'ansible_host': '54.190.184.126', 35 | } 36 | } 37 | advertised_ips = ["35.91.106.231", "35.88.129.205", "54.190.184.126"] 38 | groups = { 39 | 'connect': ['35.91.106.231', '35.88.129.205', '54.190.184.126'] 40 | } 41 | jmx_ssl = "true" 42 | 43 | # Render the template with the provided hostvars, groups, and defaults 44 | rendered_template = template.render(hostvars=hostvars, advertised_ips=advertised_ips, groups=groups, jmx_ssl=jmx_ssl, **defaults) 45 | 46 | # Define the expected bootstrap server line 47 | #expected_bootstrap_servers = 'bootstrap.servers=35.91.106.231:9092,35.88.129.205:9092,54.190.184.126:9092' 48 | 49 | # Assert that the expected line is contained within the rendered template 50 | #self.assertIn(expected_bootstrap_servers, rendered_template) 51 | 52 | 53 | print(rendered_template) 54 | 55 | if __name__ == '__main__': 56 | unittest.main() 57 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # run ansible lint using the config file 2 | .PHONY: lint 3 | lint: 4 | @echo "Running ansible-lint" 5 | @ansible-lint -c .ansible-lint 6 | 7 | # lint the binary_bundler role in roles/binary_bundler 8 | .PHONY: lint-binary_bundler 9 | lint-binary_bundler: 10 | @echo "Running ansible-lint on binary_bundler role" 11 | @ansible-lint -c .ansible-lint roles/binary_bundler 12 | 13 | # lint the client config role in roles/client_config 14 | .PHONY: lint-client_config 15 | lint-client_config: 16 | @echo "Running ansible-lint on client_config role" 17 | @ansible-lint -c .ansible-lint roles/client_config 18 | 19 | # lint the demo_certs role in roles/demo_certs 20 | .PHONY: lint-demo_certs 21 | lint-demo_certs: 22 | @echo "Running ansible-lint on demo_certs role" 23 | @ansible-lint -c .ansible-lint roles/demo_certs 24 | 25 | # lint the redpanda_broker role in roles/redpanda_broker 26 | .PHONY: lint-redpanda_broker 27 | lint-redpanda_broker: 28 | @echo "Running ansible-lint on redpanda_broker role" 29 | @ansible-lint -c .ansible-lint roles/redpanda_broker 30 | 31 | # lint the redpanda_connect role in roles/redpanda_connect 32 | .PHONY: lint-redpanda_connect 33 | lint-redpanda_connect: 34 | @echo "Running ansible-lint on redpanda_connect role" 35 | @ansible-lint -c .ansible-lint roles/redpanda_connect 36 | 37 | # lint the redpanda_console role in roles/redpanda_console 38 | .PHONY: lint-redpanda_console 39 | lint-redpanda_console: 40 | @echo "Running ansible-lint on redpanda_console role" 41 | @ansible-lint -c .ansible-lint roles/redpanda_console 42 | 43 | # lint the systemctl_setup role in roles/systemctl_setup 44 | .PHONY: lint-systemctl_setup 45 | lint-sysctl_setup: 46 | @echo "Running ansible-lint on systemctl_setup role" 47 | @ansible-lint -c .ansible-lint roles/sysctl_setup 48 | 49 | # lint the system_setup role in roles/system_setup 50 | .PHONY: lint-system_setup 51 | lint-system_setup: 52 | @echo "Running ansible-lint on system_setup role" 53 | @ansible-lint -c .ansible-lint roles/system_setup 54 | 55 | ANSIBLE_GALAXY_API_KEY ?= $(shell bash -c 'read -p "Enter your API key: " api_key; echo $$api_key') 56 | .PHONY: publish 57 | publish: 58 | rm -f redpanda-cluster-*.tar.gz 2>/dev/null && \ 59 | ansible-galaxy collection build && \ 60 | ansible-galaxy collection publish redpanda-cluster-*.tar.gz --token $(ANSIBLE_GALAXY_API_KEY) -s https://galaxy.ansible.com/api/ && \ 61 | rm -f redpanda-cluster-*.tar.gz 2>/dev/null 62 | -------------------------------------------------------------------------------- /roles/redpanda_console/tasks/configure-rpm-repository.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set Redpanda RPM and GPG Key details 3 | ansible.builtin.set_fact: 4 | rp_key_rpm_actual: "{{ rp_key_rpm }}" 5 | rp_standard_rpm_actual: "{{ rp_standard_rpm }}" 6 | rp_noarch_rpm_actual: "{{ rp_noarch_rpm }}" 7 | rp_source_rpm_actual: "{{ rp_source_rpm }}" 8 | repo_name_prefix: "redpanda-redpanda" 9 | 10 | - name: Add redpanda-redpanda RPM repository 11 | ansible.builtin.yum_repository: 12 | name: "{{ repo_name_prefix }}" 13 | description: "{{ repo_name_prefix }}" 14 | baseurl: "{{ rp_standard_rpm_actual }}" 15 | gpgkey: "{{ rp_key_rpm_actual }}" 16 | repo_gpgcheck: true 17 | gpgcheck: true 18 | enabled: true 19 | sslcacert: '/etc/pki/tls/certs/ca-bundle.crt' 20 | sslverify: true 21 | metadata_expire: 300 22 | skip_if_unavailable: true 23 | proxy: "{{ rpm_proxy | default('_none_') }}" 24 | 25 | - name: Add redpanda-redpanda-noarch RPM repository 26 | ansible.builtin.yum_repository: 27 | name: "{{ repo_name_prefix }}-noarch" 28 | description: "{{ repo_name_prefix }}-noarch" 29 | baseurl: "{{ rp_noarch_rpm_actual }}" 30 | gpgkey: "{{ rp_key_rpm_actual }}" 31 | repo_gpgcheck: true 32 | gpgcheck: true 33 | enabled: true 34 | sslcacert: '/etc/pki/tls/certs/ca-bundle.crt' 35 | sslverify: true 36 | metadata_expire: 300 37 | skip_if_unavailable: true 38 | proxy: "{{ rpm_proxy | default('_none_') }}" 39 | 40 | - name: Add redpanda-redpanda-source RPM repository 41 | ansible.builtin.yum_repository: 42 | name: "{{ repo_name_prefix }}-source" 43 | description: "{{ repo_name_prefix }}-source" 44 | baseurl: "{{ rp_source_rpm_actual }}" 45 | gpgkey: "{{ rp_key_rpm_actual }}" 46 | repo_gpgcheck: true 47 | gpgcheck: true 48 | enabled: true 49 | sslcacert: '/etc/pki/tls/certs/ca-bundle.crt' 50 | sslverify: true 51 | metadata_expire: 300 52 | skip_if_unavailable: true 53 | proxy: "{{ rpm_proxy | default('_none_') }}" 54 | 55 | - name: Install GPG key with proxy 56 | environment: 57 | https_proxy: "{{ rpm_proxy | default('') }}" 58 | ansible.builtin.rpm_key: 59 | state: present 60 | key: "{{ rp_key_rpm_actual }}" 61 | when: https_proxy_value is defined and https_proxy_value | length > 0 62 | 63 | - name: Install GPG key 64 | ansible.builtin.rpm_key: 65 | state: present 66 | key: "{{ rp_key_rpm_actual }}" 67 | when: (https_proxy_value is not defined or https_proxy_value | length == 0) 68 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/install-rp-rpm-airgap.yml: -------------------------------------------------------------------------------- 1 | - name: Check if RP exists 2 | ansible.builtin.stat: 3 | path: "/usr/bin/redpanda" 4 | register: rp_exists_check 5 | 6 | - name: Get version from file 7 | ansible.builtin.command: 8 | cmd: "/usr/bin/redpanda --version" 9 | register: rp_ver_output 10 | when: rp_exists_check.stat.exists 11 | 12 | - name: redpanda_version is post packaging split or not 13 | ansible.builtin.set_fact: 14 | is_post_split: "{{ redpanda_version == 'latest' or redpanda_version >= '24.2' }}" 15 | 16 | - name: Check if redpanda_version is set to latest 17 | ansible.builtin.set_fact: 18 | needs_update: true 19 | when: redpanda_version == "latest" 20 | 21 | - name: Compare version if redpanda_version is not set to latest 22 | ansible.builtin.set_fact: 23 | needs_update: "{{ (redpanda_version.replace('v', '').split('-')[0] | string ) is version((rp_ver_output.stdout.replace('v', '').split('-')[0] | string), '>') }}" 24 | when: 25 | - rp_exists_check.stat.exists 26 | - redpanda_version != "latest" 27 | 28 | - name: Copy Redpanda RPMs tarball to host 29 | ansible.builtin.copy: 30 | src: "{{ airgap_copy_src }}/redpanda_rpms.tar.gz" 31 | dest: "{{ airgap_copy_dest }}/redpanda_rpms.tar.gz" 32 | mode: '0644' 33 | tags: 34 | - airgap-tarball-install 35 | when: 36 | - (needs_update | default(false)) or not rp_exists_check.stat.exists 37 | 38 | - name: Unpack Redpanda RPMs on host 39 | ansible.builtin.unarchive: 40 | src: "{{ airgap_copy_dest }}/redpanda_rpms.tar.gz" 41 | dest: "{{ airgap_copy_dest }}" 42 | tags: 43 | - airgap-tarball-install 44 | when: 45 | - (needs_update | default(false)) or not rp_exists_check.stat.exists 46 | 47 | - name: Install Redpanda RPMs (pre package split) 48 | ansible.builtin.shell: "rpm -Uvh --force {{ airgap_copy_dest }}/*.rpm" 49 | register: rpm_result 50 | failed_when: rpm_result.rc != 2 and rpm_result.rc != 0 51 | become: true 52 | tags: 53 | - airgap-tarball-install 54 | when: 55 | - (needs_update | default(false)) or not rp_exists_check.stat.exists 56 | - not is_post_split 57 | 58 | - name: Install Redpanda RPMs (post package split) 59 | ansible.builtin.shell: "rpm -Uvh --force {{ airgap_copy_dest }}/{{item}}*__*.rpm" 60 | register: rpm_result 61 | failed_when: rpm_result.rc != 2 and rpm_result.rc != 0 62 | become: true 63 | tags: 64 | - airgap-tarball-install 65 | when: 66 | - (needs_update | default(false)) or not rp_exists_check.stat.exists 67 | - is_post_split 68 | loop: 69 | - redpanda-rpk 70 | - redpanda-tuner 71 | - redpanda 72 | 73 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure redpanda_version is defined 3 | ansible.builtin.assert: 4 | that: 5 | - redpanda_version != '' 6 | fail_msg: "Variable 'redpanda_version' must be defined!" 7 | when: not (install_certs_only | default(false) | bool) 8 | 9 | # Copies certs set in ca_cert_file and node_cert_file into the redpanda nodes. see defaults for the default values 10 | # only necessary when TLS is enabled 11 | - name: Install certs 12 | ansible.builtin.include_tasks: install-certs.yml 13 | when: 14 | - handle_cert_install | default(false) | bool 15 | 16 | # End play if install_certs_only is true 17 | - name: End play if only installing certs 18 | ansible.builtin.meta: end_play 19 | when: install_certs_only | default(false) | bool 20 | 21 | - name: Configure Redpanda RPM Repository 22 | ansible.builtin.include_tasks: configure-rpm-repository.yml 23 | when: 24 | - ansible_os_family == 'RedHat' 25 | - not (enable_airgap | bool) 26 | - not (development_build | bool) 27 | 28 | - name: Configure Redpanda DEB Repository 29 | ansible.builtin.include_tasks: configure-deb-repository.yml 30 | when: 31 | - ansible_os_family == 'Debian' 32 | - not (enable_airgap | bool) 33 | - not (development_build | bool) 34 | 35 | - name: Configure Redpanda DEB Repository Development 36 | ansible.builtin.include_tasks: install-nightly-build-deb.yml 37 | when: 38 | - ansible_os_family == 'Debian' 39 | - not (enable_airgap | bool) 40 | - development_build | bool 41 | 42 | - name: Configure Redpanda DEB Repository Development 43 | ansible.builtin.include_tasks: install-nightly-build-rpm.yml 44 | when: 45 | - ansible_os_family == 'RedHat' 46 | - not (enable_airgap | bool) 47 | - development_build | bool 48 | 49 | - name: Install Redpanda RPM 50 | ansible.builtin.include_tasks: install-rp-rpm.yml 51 | when: 52 | - ansible_os_family == 'RedHat' 53 | - not (enable_airgap | bool) 54 | 55 | - name: Install Redpanda DEB 56 | ansible.builtin.include_tasks: install-rp-deb.yml 57 | when: 58 | - ansible_os_family == 'Debian' 59 | - not (enable_airgap | bool) 60 | 61 | - name: Install Redpanda RPM Airgap 62 | ansible.builtin.include_tasks: install-rp-rpm-airgap.yml 63 | when: 64 | - ansible_os_family == 'RedHat' 65 | - enable_airgap | bool 66 | 67 | - name: Install Redpanda DEB Airgap 68 | ansible.builtin.include_tasks: install-rp-deb-airgap.yml 69 | when: 70 | - ansible_os_family == 'Debian' 71 | - enable_airgap | bool 72 | 73 | - name: Start Redpanda 74 | ansible.builtin.include_tasks: start-redpanda.yml 75 | -------------------------------------------------------------------------------- /roles/demo_certs/tasks/generate-csrs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Test hosts list 3 | tags: 4 | - generate_csrs 5 | ansible.builtin.debug: 6 | msg: 7 | - "ipv4 : {{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" 8 | - "private ip : {{ hostvars[inventory_hostname]['private_ip'] }}" 9 | - "ansible_hostname: {{ hostvars[inventory_hostname]['ansible_hostname'] }}" 10 | - "ansible_fqdn : {{ hostvars[inventory_hostname]['ansible_fqdn'] }}" 11 | 12 | - name: Create redpanda group 13 | ansible.builtin.group: 14 | name: "{{ redpanda_group }}" 15 | state: present 16 | system: true 17 | become: true 18 | 19 | 20 | - name: Create redpanda user if it doesn't exist already 21 | tags: 22 | - generate_csrs 23 | ansible.builtin.user: 24 | name: "{{ redpanda_user }}" 25 | group: "{{ redpanda_group }}" 26 | system: true 27 | shell: /usr/sbin/nologin 28 | create_home: false 29 | state: present 30 | become: true 31 | 32 | - name: Ensure /etc/redpanda/certs exists 33 | tags: 34 | - generate_csrs 35 | ansible.builtin.file: 36 | path: "{{ redpanda_certs_dir }}" 37 | state: directory 38 | owner: redpanda 39 | group: redpanda 40 | mode: "0755" 41 | 42 | - name: Copy node template 43 | tags: 44 | - generate_csrs 45 | ansible.builtin.template: 46 | src: node.conf.tpl 47 | dest: "{{ redpanda_certs_dir }}/node.conf" 48 | owner: redpanda 49 | group: redpanda 50 | mode: "0644" 51 | 52 | - name: Install openssl 53 | tags: 54 | - generate_csrs 55 | ansible.builtin.package: 56 | name: openssl 57 | state: present 58 | 59 | - name: Generate an OpenSSH keypair on {{ansible_hostname}} 60 | tags: 61 | - generate_csrs 62 | ansible.builtin.command: 63 | creates: "{{ redpanda_key_file }}" 64 | chdir: "{{ redpanda_certs_dir }}" 65 | cmd: openssl genrsa -out node.key 2048 66 | 67 | - name: Generate a Certificate Signing Request on {{ansible_hostname}} 68 | tags: 69 | - generate_csrs 70 | ansible.builtin.command: 71 | creates: "{{ redpanda_csr_file }}" 72 | chdir: "{{ redpanda_certs_dir }}" 73 | cmd: openssl req -new -config node.conf -key node.key -out node.csr -batch 74 | 75 | - name: CHOWN to redpanda 76 | tags: 77 | - generate_csrs 78 | ansible.builtin.file: 79 | path: "{{ redpanda_certs_dir }}" 80 | state: directory 81 | owner: redpanda 82 | group: redpanda 83 | recurse: true 84 | 85 | - name: Fetch CSRs 86 | tags: 87 | - generate_csrs 88 | ansible.builtin.fetch: 89 | src: "{{ redpanda_csr_file }}" 90 | dest: tls/certs/{{ansible_hostname}}/node.csr 91 | flat: true 92 | -------------------------------------------------------------------------------- /roles/redpanda_console/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | enable_airgap: false 3 | 4 | redpanda_admin_api_port: 9644 5 | redpanda_kafka_port: 9092 6 | redpanda_schema_registry_port: 8081 7 | kafka_connect_port: 8083 8 | 9 | http_port: 8080 10 | https_port: 443 11 | 12 | redpanda_group: redpanda 13 | redpanda_user: redpanda 14 | 15 | redpanda_base_url: "https://dl.redpanda.com" 16 | 17 | redpanda_install_status: present # If redpanda_version is set to latest, changing redpanda_install_status to latest will effect an upgrade, otherwise the currently installed version will remain 18 | 19 | redpanda_base_dir: /var/lib/redpanda 20 | redpanda_data_directory: "{{ redpanda_base_dir }}/data" 21 | 22 | redpanda_certs_dir: /etc/redpanda/certs 23 | redpanda_csr_file: "{{ redpanda_certs_dir }}/node.csr" 24 | redpanda_key_file: "{{ redpanda_certs_dir }}/node.key" 25 | redpanda_cert_file: "{{ redpanda_certs_dir }}/node.crt" 26 | redpanda_truststore_file: "{{ redpanda_certs_dir }}/truststore.pem" 27 | 28 | # We default to use the same redpanda dirs, but in prod, a user can specify them directly 29 | console_certs_dir: "{{ redpanda_certs_dir }}" 30 | console_key_file: "{{ console_certs_dir }}/node.key" 31 | console_cert_file: "{{ console_certs_dir }}/node.crt" 32 | 33 | # the rpms 34 | rp_key_rpm: "{{ redpanda_base_url }}/public/redpanda/gpg.988A7B0A4918BC85.key" 35 | rp_standard_rpm: "{{ redpanda_base_url }}/public/redpanda/rpm/{{ rp_rpm_distro_map[ansible_distribution] | default('el') }}/{{ ansible_distribution_major_version }}/$basearch" 36 | rp_noarch_rpm: "{{ redpanda_base_url }}/public/redpanda/rpm/{{ rp_rpm_distro_map[ansible_distribution] | default('el') }}/{{ ansible_distribution_major_version }}/noarch" 37 | rp_source_rpm: "{{ redpanda_base_url }}/public/redpanda/rpm/{{ rp_rpm_distro_map[ansible_distribution] | default('el') }}/{{ ansible_distribution_major_version }}/SRPMS" 38 | rp_rpm_distro_map: 39 | Fedora: fedora 40 | Amazon: amzn 41 | default: el 42 | 43 | 44 | # the debs 45 | rp_key_deb: "{{ redpanda_base_url }}/sMIXnoa7DK12JW4A/redpanda/gpg.988A7B0A4918BC85.key" 46 | rp_key_deb_unstable: "{{ redpanda_base_url }}/E4xN1tVe3Xy60GTx/redpanda-unstable/gpg.39C20393EC2E8747.key" 47 | rp_key_path_deb: "/usr/share/keyrings/redpanda-redpanda-archive-keyring.gpg" 48 | rp_repo_signing_deb: "deb [signed-by=/usr/share/keyrings/redpanda-redpanda-archive-keyring.gpg] {{ redpanda_base_url }}/public/redpanda/deb/{{ansible_distribution | lower}} {{ ansible_distribution_release | lower }} main" 49 | rp_repo_signing_src_deb: "deb-src [signed-by=/usr/share/keyrings/redpanda-redpanda-archive-keyring.gpg] {{ redpanda_base_url }}/public/redpanda/deb/{{ansible_distribution | lower}} {{ ansible_distribution_release | lower }} main" 50 | -------------------------------------------------------------------------------- /roles/demo_certs/tasks/generate-csrs-keystore.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Test hosts list 3 | tags: 4 | - generate_csrs_keystore 5 | ansible.builtin.debug: 6 | msg: 7 | - "ipv4 : {{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" 8 | - "private ip : {{ hostvars[inventory_hostname]['private_ip'] }}" 9 | - "ansible_hostname: {{ hostvars[inventory_hostname]['ansible_hostname'] }}" 10 | - "ansible_fqdn : {{ hostvars[inventory_hostname]['ansible_fqdn'] }}" 11 | 12 | - name: Create redpanda group 13 | ansible.builtin.group: 14 | name: "{{ redpanda_group }}" 15 | state: present 16 | system: true 17 | become: true 18 | 19 | - name: Create redpanda user if it doesn't exist already 20 | tags: 21 | - generate_csrs_keystore 22 | ansible.builtin.user: 23 | name: "{{ redpanda_user }}" 24 | group: "{{ redpanda_group }}" 25 | system: true 26 | shell: /usr/sbin/nologin 27 | create_home: false 28 | state: present 29 | become: true 30 | 31 | - name: Ensure /etc/redpanda/certs exists 32 | tags: 33 | - generate_csrs_keystore 34 | ansible.builtin.file: 35 | path: "{{ redpanda_certs_dir }}" 36 | state: directory 37 | owner: redpanda 38 | group: redpanda 39 | mode: "0755" 40 | 41 | - name: Copy node template 42 | tags: 43 | - generate_csrs_keystore 44 | ansible.builtin.template: 45 | src: node.conf.tpl 46 | dest: "{{ redpanda_certs_dir }}/node.conf" 47 | owner: redpanda 48 | group: redpanda 49 | mode: "0644" 50 | 51 | - name: Install openssl 52 | tags: 53 | - generate_csrs_keystore 54 | ansible.builtin.package: 55 | name: openssl 56 | state: present 57 | 58 | - name: Generate an OpenSSH keypair on {{ansible_hostname}} 59 | tags: 60 | - generate_csrs_keystore 61 | ansible.builtin.command: 62 | creates: "{{ redpanda_key_file }}" 63 | chdir: "{{ redpanda_certs_dir }}" 64 | cmd: openssl genrsa -out node.key 2048 65 | 66 | - name: Generate a Certificate Signing Request on {{ansible_hostname}} 67 | tags: 68 | - generate_csrs_keystore 69 | ansible.builtin.command: 70 | creates: "{{ redpanda_csr_file }}" 71 | chdir: "{{ redpanda_certs_dir }}" 72 | cmd: openssl req -new -config node.conf -key node.key -out node.csr -batch 73 | 74 | - name: CHOWN to redpanda 75 | tags: 76 | - generate_csrs_keystore 77 | ansible.builtin.file: 78 | path: "{{ redpanda_certs_dir }}" 79 | state: directory 80 | owner: redpanda 81 | group: redpanda 82 | recurse: true 83 | 84 | - name: Fetch CSRs 85 | tags: 86 | - generate_csrs_keystore 87 | ansible.builtin.fetch: 88 | src: "{{ redpanda_csr_file }}" 89 | dest: tls/certs/{{ansible_hostname}}/node.csr 90 | flat: true 91 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/install-rp-deb-airgap.yml: -------------------------------------------------------------------------------- 1 | - name: Check if RP exists 2 | ansible.builtin.stat: 3 | path: "/usr/bin/redpanda" 4 | register: rp_exists_check 5 | 6 | - name: Get version from file 7 | ansible.builtin.command: 8 | cmd: "/usr/bin/redpanda --version" 9 | register: rp_ver_output 10 | when: rp_exists_check.stat.exists 11 | 12 | - name: redpanda_version is post packaging split or not 13 | ansible.builtin.set_fact: 14 | is_post_split: "{{ redpanda_version == 'latest' or redpanda_version >= '24.2' }}" 15 | 16 | - name: If redpanda_version is latest set needs_update to true 17 | ansible.builtin.set_fact: 18 | needs_update: true 19 | when: redpanda_version == "latest" 20 | 21 | - name: Compare version if redpanda_version is not set to latest 22 | ansible.builtin.set_fact: 23 | needs_update: "{{ (redpanda_version.replace('v', '').split('-')[0] | string ) is version((rp_ver_output.stdout.replace('v', '').split('-')[0] | string), '>') }}" 24 | when: 25 | - rp_exists_check.stat.exists 26 | - redpanda_version != "latest" 27 | 28 | - name: Copy Redpanda DEBs tarball to host 29 | ansible.builtin.copy: 30 | src: "{{ airgap_copy_src }}/redpanda_debs.tar.gz" 31 | dest: "{{ airgap_copy_dest }}/redpanda_debs.tar.gz" 32 | mode: '0644' 33 | tags: 34 | - airgap-tarball-install 35 | when: 36 | - (needs_update | default(false)) or not rp_exists_check.stat.exists 37 | 38 | - name: Unpack Redpanda DEBs on host 39 | ansible.builtin.unarchive: 40 | src: "{{ airgap_copy_dest }}/redpanda_debs.tar.gz" 41 | dest: "{{ airgap_copy_dest }}" 42 | tags: 43 | - airgap-tarball-install 44 | when: 45 | - (needs_update | default(false)) or not rp_exists_check.stat.exists 46 | 47 | - name: Install Redpanda DEBs (pre-package split) 48 | ansible.builtin.shell: "dpkg -i --force-confold {{ airgap_copy_dest }}/redpanda_*.deb" 49 | become: true 50 | tags: 51 | - airgap-tarball-install 52 | when: 53 | - (needs_update | default(false)) or not rp_exists_check.stat.exists 54 | - not is_post_split 55 | 56 | - name: Install Redpanda DEBs (post-package split) 57 | ansible.builtin.shell: "dpkg -i --force-confold {{ airgap_copy_dest }}/{{ item }}_*.deb" 58 | become: true 59 | tags: 60 | - airgap-tarball-install 61 | when: 62 | - ((needs_update | default(false)) or not rp_exists_check.stat.exists) 63 | - is_post_split 64 | loop: 65 | - redpanda-rpk 66 | - redpanda-tuner 67 | - redpanda 68 | 69 | - name: Ensure /var/lib/redpanda and all contents are owned by redpanda:redpanda 70 | become: true 71 | ansible.builtin.file: 72 | path: "/var/lib/redpanda" 73 | state: directory 74 | recurse: true 75 | owner: redpanda 76 | group: redpanda 77 | mode: '0755' 78 | tags: 79 | - airgap-tarball-install 80 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tasks/configure-rpm-repository.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set Redpanda RPM and GPG Key details 3 | ansible.builtin.set_fact: 4 | rp_key_rpm_actual: "{{ is_using_unstable | bool | default(false) | ternary(rp_key_rpm_unstable, rp_key_rpm) }}" 5 | rp_standard_rpm_actual: "{{ is_using_unstable | bool | default(false) | ternary(rp_standard_rpm_unstable, rp_standard_rpm) }}" 6 | rp_noarch_rpm_actual: "{{ is_using_unstable | bool | default(false) | ternary(rp_noarch_rpm_unstable, rp_noarch_rpm) }}" 7 | rp_source_rpm_actual: "{{ is_using_unstable | bool | default(false) | ternary(rp_source_rpm_unstable, rp_source_rpm) }}" 8 | repo_name_prefix: "{{ is_using_unstable | bool | default(false) | ternary('redpanda-redpanda-unstable', 'redpanda-redpanda') }}" 9 | 10 | - name: Add redpanda-redpanda RPM repository 11 | ansible.builtin.yum_repository: 12 | name: "{{ repo_name_prefix }}" 13 | description: "{{ repo_name_prefix }}" 14 | baseurl: "{{ rp_standard_rpm_actual }}" 15 | gpgkey: "{{ rp_key_rpm_actual }}" 16 | repo_gpgcheck: true 17 | gpgcheck: true 18 | enabled: true 19 | sslcacert: '/etc/pki/tls/certs/ca-bundle.crt' 20 | sslverify: true 21 | metadata_expire: 300 22 | skip_if_unavailable: true 23 | proxy: "{{ rpm_proxy | default('_none_') }}" 24 | 25 | - name: Add redpanda-redpanda-noarch RPM repository 26 | ansible.builtin.yum_repository: 27 | name: "{{ repo_name_prefix }}-noarch" 28 | description: "{{ repo_name_prefix }}-noarch" 29 | baseurl: "{{ rp_noarch_rpm_actual }}" 30 | gpgkey: "{{ rp_key_rpm_actual }}" 31 | repo_gpgcheck: true 32 | gpgcheck: true 33 | enabled: true 34 | sslcacert: '/etc/pki/tls/certs/ca-bundle.crt' 35 | sslverify: true 36 | metadata_expire: 300 37 | skip_if_unavailable: true 38 | proxy: "{{ rpm_proxy | default('_none_') }}" 39 | 40 | - name: Add redpanda-redpanda-source RPM repository 41 | ansible.builtin.yum_repository: 42 | name: "{{ repo_name_prefix }}-source" 43 | description: "{{ repo_name_prefix }}-source" 44 | baseurl: "{{ rp_source_rpm_actual }}" 45 | gpgkey: "{{ rp_key_rpm_actual }}" 46 | repo_gpgcheck: true 47 | gpgcheck: true 48 | enabled: true 49 | sslcacert: '/etc/pki/tls/certs/ca-bundle.crt' 50 | sslverify: true 51 | metadata_expire: 300 52 | skip_if_unavailable: true 53 | proxy: "{{ rpm_proxy | default('_none_') }}" 54 | 55 | - name: Install GPG key with proxy 56 | environment: 57 | https_proxy: "{{ rpm_proxy | default('') }}" 58 | ansible.builtin.rpm_key: 59 | state: present 60 | key: "{{ rp_key_rpm_actual }}" 61 | when: https_proxy_value is defined and https_proxy_value | length > 0 62 | 63 | - name: Install GPG key 64 | ansible.builtin.rpm_key: 65 | state: present 66 | key: "{{ rp_key_rpm_actual }}" 67 | when: (https_proxy_value is not defined or https_proxy_value | length == 0) 68 | -------------------------------------------------------------------------------- /roles/redpanda_console/templates/defaults.j2: -------------------------------------------------------------------------------- 1 | { 2 | "kafka": { 3 | "brokers": [ 4 | {% for host in advertised_ips %} 5 | "{{ host }}:{{ redpanda_kafka_port }}"{% if not loop.last %},{% endif %} 6 | {% endfor %} 7 | ]{% if enable_tls | default(false) %}, 8 | "tls": { 9 | "enabled": true, 10 | "caFilepath": "{{ redpanda_truststore_file }}", 11 | "certFilepath": "{{ redpanda_cert_file }}", 12 | "keyFilepath": "{{ redpanda_key_file }}", 13 | "insecureSkipTlsVerify": false 14 | } 15 | {% endif %} 16 | }, 17 | "redpanda": { 18 | "adminApi": { 19 | "enabled": true, 20 | "urls": [ 21 | {% for host in advertised_ips %} 22 | "{% if enable_tls | default(false) %}https{% else %}http{% endif %}://{{ host }}:{{ redpanda_admin_api_port }}"{% if not loop.last %},{% endif %} 23 | {% endfor %} 24 | ]{% if enable_tls | default(false) %}, 25 | "tls": { 26 | "enabled": true, 27 | "caFilepath": "{{ redpanda_truststore_file }}", 28 | "certFilepath": "{{ redpanda_cert_file }}", 29 | "keyFilepath": "{{ redpanda_key_file }}", 30 | "insecureSkipTlsVerify": false 31 | } 32 | {% endif %} 33 | } 34 | },{% if enable_tls | default(false) %} 35 | "server": { 36 | "listenPort": {{ http_port }}, 37 | "httpsListenPort": {{ https_port }}, 38 | "tls": { 39 | "enabled": true, 40 | "certFilepath": "{{ console_cert_file }}", 41 | "keyFilepath": "{{ console_key_file }}" 42 | } 43 | },{% endif %}{% if kafka_connect_advertised_ips is defined %} 44 | "kafkaConnect": { 45 | "enabled": true, 46 | "clusters": [ 47 | { 48 | "name": "Connect", 49 | "url": "{% if enable_tls | default(false) %}https{% else %}http{% endif %}://{{ kafka_connect_advertised_ips[0] }}:{{ kafka_connect_port }}" 50 | {% if enable_tls | default(false) %}, 51 | "tls": { 52 | "enabled": true, 53 | "caFilepath": "{{ redpanda_truststore_file }}", 54 | "certFilepath": "{{ redpanda_cert_file }}", 55 | "keyFilepath": "{{ redpanda_key_file }}", 56 | "insecureSkipTlsVerify": false 57 | } 58 | {% endif %} 59 | } 60 | ] 61 | },{% endif %} 62 | "schemaRegistry": { 63 | "enabled": true, 64 | "urls": [ 65 | {% for host in advertised_ips %} 66 | "{% if enable_tls | default(false) %}https{% else %}http{% endif %}://{{ host }}:{{ redpanda_schema_registry_port }}"{% if not loop.last %},{% endif %} 67 | {% endfor %} 68 | ]{% if enable_tls | default(false) %}, 69 | "tls": { 70 | "enabled": true, 71 | "caFilepath": "{{ redpanda_truststore_file }}", 72 | "certFilepath": "{{ redpanda_cert_file }}", 73 | "keyFilepath": "{{ redpanda_key_file }}", 74 | "insecureSkipTlsVerify": false 75 | } 76 | {% endif %} 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /roles/redpanda_logging/tests/rsyslog_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import yaml 4 | from jinja2 import Environment, FileSystemLoader 5 | 6 | 7 | class TestRedpandaRsyslogTemplate(unittest.TestCase): 8 | def setUp(self): 9 | """Set up test environment.""" 10 | self.current_dir = os.path.dirname(os.path.abspath(__file__)) 11 | self.templates_dir = os.path.join(self.current_dir, '..', 'templates') 12 | self.defaults_file = os.path.join(self.current_dir, '..', 'defaults', 'main.yml') 13 | 14 | # Load defaults 15 | with open(self.defaults_file, 'r') as f: 16 | self.defaults = yaml.safe_load(f) 17 | 18 | # Create Jinja2 environment 19 | self.env = Environment(loader=FileSystemLoader(self.templates_dir)) 20 | 21 | def test_rsyslog_default_config(self): 22 | """Test rsyslog config with default values.""" 23 | template = self.env.get_template('redpanda-rsyslog.conf.j2') 24 | 25 | # Test data with default values 26 | context = { 27 | 'redpanda_logging_log_file': '/var/log/redpanda.log', 28 | 'redpanda_logging_program': 'rpk' 29 | } 30 | 31 | rendered_template = template.render(**context) 32 | 33 | # Verify default configuration 34 | self.assertIn("if $programname == 'rpk' then /var/log/redpanda.log", rendered_template) 35 | self.assertIn('& stop', rendered_template) 36 | self.assertEqual(rendered_template.count('& stop'), 1) 37 | 38 | def test_rsyslog_custom_path(self): 39 | """Test rsyslog config with custom log path.""" 40 | template = self.env.get_template('redpanda-rsyslog.conf.j2') 41 | 42 | # Test data with custom path 43 | context = { 44 | 'redpanda_logging_log_file': '/app/logs/redpanda.log', 45 | 'redpanda_logging_program': 'rpk' 46 | } 47 | 48 | rendered_template = template.render(**context) 49 | 50 | # Verify custom path is used 51 | self.assertIn("if $programname == 'rpk' then /app/logs/redpanda.log", rendered_template) 52 | self.assertIn('& stop', rendered_template) 53 | 54 | def test_rsyslog_custom_program_name(self): 55 | """Test rsyslog config with custom program name.""" 56 | template = self.env.get_template('redpanda-rsyslog.conf.j2') 57 | 58 | # Test data with custom program name 59 | context = { 60 | 'redpanda_logging_log_file': '/var/log/redpanda.log', 61 | 'redpanda_logging_program': 'redpanda-broker' 62 | } 63 | 64 | rendered_template = template.render(**context) 65 | 66 | # Verify custom program name is used 67 | self.assertIn("if $programname == 'redpanda-broker' then /var/log/redpanda.log", rendered_template) 68 | self.assertIn('& stop', rendered_template) 69 | 70 | 71 | if __name__ == '__main__': 72 | unittest.main() -------------------------------------------------------------------------------- /roles/redpanda_broker/tests/defaults_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import yaml 4 | from jinja2 import Environment, FileSystemLoader 5 | 6 | class TestRedpandaTemplate(unittest.TestCase): 7 | def setUp(self): 8 | # Get the absolute path of the current file 9 | self.current_dir = os.path.dirname(os.path.abspath(__file__)) 10 | 11 | # Construct the path to the templates directory 12 | self.templates_dir = os.path.join(self.current_dir, '..', 'templates/configs') 13 | 14 | # Construct the path to the defaults/main.yml file 15 | self.defaults_file = os.path.join(self.current_dir, '..', 'defaults', 'main.yml') 16 | 17 | # Load the defaults from the YAML file 18 | with open(self.defaults_file, 'r') as f: 19 | self.defaults = yaml.safe_load(f) 20 | 21 | # Create a Jinja2 environment 22 | self.env = Environment(loader=FileSystemLoader(self.templates_dir)) 23 | 24 | # Define the hostvars and groups for rendering the template 25 | self.hostvars = { 26 | '35.91.106.231': { 27 | 'ansible_host': '35.91.106.231', 28 | 'advertised_ip': '35.91.106.231', 29 | 'private_ip': '192.168.1.1' 30 | }, 31 | '35.88.129.205': { 32 | 'ansible_host': '35.88.129.205', 33 | 'advertised_ip': '35.88.129.205', 34 | 'private_ip': '192.168.1.2' 35 | }, 36 | '54.190.184.126': { 37 | 'ansible_host': '54.190.184.126', 38 | 'advertised_ip': '54.190.184.126', 39 | 'private_ip': '192.168.1.3' 40 | } 41 | } 42 | self.groups = { 43 | 'redpanda': ['35.91.106.231', '35.88.129.205', '54.190.184.126'] 44 | } 45 | 46 | def test_redpanda_template_default_listener(self): 47 | # Load the template from file 48 | template = self.env.get_template('defaults.j2') 49 | 50 | # Render the template with the provided hostvars, groups, and defaults without multiple_advertised_kafka_listeners 51 | rendered_template = template.render( 52 | hostvars=self.hostvars, 53 | groups=self.groups, 54 | inventory_hostname='35.91.106.231', 55 | **self.defaults 56 | ) 57 | 58 | # Define the expected default advertised_kafka_api value 59 | expected_advertised_kafka_api_default = { 60 | "address": "35.91.106.231", 61 | "port": str(self.defaults['redpanda_kafka_port']) 62 | } 63 | 64 | # Convert the rendered template to a dictionary for easier comparison 65 | rendered_dict = yaml.safe_load(rendered_template) 66 | 67 | # Assert that the expected default value is in the rendered template 68 | self.assertIn(expected_advertised_kafka_api_default, rendered_dict['node']['redpanda']['advertised_kafka_api']) 69 | 70 | print(rendered_template) 71 | 72 | if __name__ == '__main__': 73 | unittest.main() 74 | -------------------------------------------------------------------------------- /roles/redpanda_console/templates/pre_v3_defaults.j2: -------------------------------------------------------------------------------- 1 | { 2 | "kafka": { 3 | "brokers": [ 4 | {% for host in advertised_ips %} 5 | "{{ host }}:{{ redpanda_kafka_port }}"{% if not loop.last %},{% endif %} 6 | {% endfor %} 7 | ], 8 | "schemaRegistry": { 9 | "enabled": true, 10 | "urls": [ 11 | {% for host in advertised_ips %} 12 | "{% if enable_tls | default(false) %}https{% else %}http{% endif %}://{{ host }}:{{ redpanda_schema_registry_port }}"{% if not loop.last %},{% endif %} 13 | {% endfor %} 14 | ]{% if enable_tls | default(false) %}, 15 | "tls": { 16 | "enabled": true, 17 | "caFilepath": "{{ redpanda_truststore_file }}", 18 | "certFilepath": "{{ redpanda_cert_file }}", 19 | "keyFilepath": "{{ redpanda_key_file }}", 20 | "insecureSkipTlsVerify": false 21 | }{% endif %} 22 | }, 23 | "protobuf": { 24 | "enabled": true, 25 | "schemaRegistry": { 26 | "enabled": true, 27 | "refreshInterval": "5m" 28 | } 29 | }{% if enable_tls | default(false) %}, 30 | "tls": { 31 | "enabled": true, 32 | "caFilepath": "{{ redpanda_truststore_file }}", 33 | "certFilepath": "{{ redpanda_cert_file }}", 34 | "keyFilepath": "{{ redpanda_key_file }}", 35 | "insecureSkipTlsVerify": false 36 | } 37 | {% endif %} 38 | }, 39 | "redpanda": { 40 | "adminApi": { 41 | "enabled": true, 42 | "urls": [ 43 | {% for host in advertised_ips %} 44 | "{% if enable_tls | default(false) %}https{% else %}http{% endif %}://{{ host }}:{{ redpanda_admin_api_port }}"{% if not loop.last %},{% endif %} 45 | {% endfor %} 46 | ]{% if enable_tls | default(false) %}, 47 | "tls": { 48 | "enabled": true, 49 | "caFilepath": "{{ redpanda_truststore_file }}", 50 | "certFilepath": "{{ redpanda_cert_file }}", 51 | "keyFilepath": "{{ redpanda_key_file }}", 52 | "insecureSkipTlsVerify": false 53 | } 54 | {% endif %} 55 | } 56 | }{% if enable_tls | default(false) %}, 57 | "server": { 58 | "listenPort": {{ http_port }}, 59 | "httpsListenPort": {{ https_port }}, 60 | "tls": { 61 | "enabled": true, 62 | "certFilepath": "{{ console_cert_file }}", 63 | "keyFilepath": "{{ console_key_file }}" 64 | } 65 | }{% endif %}{% if kafka_connect_advertised_ips is defined %}, 66 | "connect": { 67 | "enabled": true, 68 | "clusters": 69 | [ 70 | { 71 | "name": "Connect", 72 | "url": "{% if enable_tls | default(false) %}https{% else %}http{% endif %}://{{ kafka_connect_advertised_ips[0] }}:{{ kafka_connect_port }}" 73 | {% if enable_tls | default(false) %}, 74 | "tls": { 75 | "enabled": true, 76 | "caFilepath": "{{ redpanda_truststore_file }}", 77 | "certFilepath": "{{ redpanda_cert_file }}", 78 | "keyFilepath": "{{ redpanda_key_file }}", 79 | "insecureSkipTlsVerify": false 80 | } 81 | {% endif %} 82 | }] 83 | }{% endif %} 84 | } 85 | -------------------------------------------------------------------------------- /roles/redpanda_broker/README.md: -------------------------------------------------------------------------------- 1 | # Ansible Deployment for Redpanda 2 | 3 | Ansible configuration to easily provision a [Redpanda](https://www.redpanda.com/) cluster. 4 | 5 | ## Installation Prerequisites 6 | 7 | Here are some prerequisites you'll need to install to run the content in this repo. You can also choose to use our 8 | Dockerfile_FEDORA or Dockerfile_UBUNTU dockerfiles to build a local client if you'd rather not install terraform and 9 | ansible on your machine. 10 | 11 | * Install Ansible: https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html 12 | * Depending on your system, you might need to install some python packages (e.g. `selinux` or `jmespath`). Ansible will 13 | throw an error with the expected python packages, both on local and remote machines. 14 | 15 | ### On Mac OS X: 16 | 17 | You can use brew to install the prerequisites. You will also need to install gnu-tar: 18 | 19 | ```commandline 20 | brew install ansible 21 | brew install gnu-tar 22 | ``` 23 | 24 | ## Basic Usage: 25 | 26 | ```shell 27 | # Set required ansible variables 28 | export ANSIBLE_COLLECTIONS_PATHS=${PWD}/artifacts/collections 29 | export ANSIBLE_ROLES_PATH=${PWD}/artifacts/roles 30 | export ANSIBLE_INVENTORY=${PWD}/${CLOUD_PROVIDER}/hosts.ini 31 | 32 | # Generate a hosts file that fits our standard 33 | 34 | # Install collections and roles 35 | ansible-galaxy install -r ./requirements.yml 36 | 37 | # Run a Playbook 38 | # You need to pick the correct playbook for you, in this case we picked provision-basic-cluster 39 | ansible-playbook ansible/provision-basic-cluster.yml --private-key ~/.ssh/id_rsa 40 | ``` 41 | 42 | ## Additional Documentation 43 | 44 | More information on consuming this collection 45 | is [available here](https://docs.redpanda.com/docs/deploy/deployment-option/self-hosted/manual/production/production-deployment-automation/) 46 | in our official documentation. 47 | 48 | ## Troubleshooting 49 | 50 | ### On Mac OS X, Python unable to fork workers 51 | 52 | If you see something like this: 53 | 54 | ``` 55 | ok: [34.209.26.177] => {“changed”: false, “stat”: {“exists”: false}} 56 | objc[57889]: +[__NSCFConstantString initialize] may have been in progress in another thread when fork() was called. 57 | objc[57889]: +[__NSCFConstantString initialize] may have been in progress in another thread when fork() was called. We cannot safely call it or ignore it in the fork() child process. Crashing instead. Set a breakpoint on objc_initializeAfterForkError to debug. 58 | ERROR! A worker was found in a dead state 59 | ``` 60 | 61 | You might try resolving by setting an environment variable: 62 | `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` 63 | 64 | See: https://stackoverflow.com/questions/50168647/multiprocessing-causes-python-to-crash-and-gives-an-error-may-have-been-in-progr 65 | 66 | ## Contribution Guide 67 | 68 | ### testing with a specific branch of redpanda-ansible-collection 69 | 70 | Change the redpanda.cluster entry in your requirements.yml file to the following: 71 | 72 | ```yaml 73 | - name: https://github.com/redpanda-data/redpanda-ansible-collection.git 74 | type: git 75 | version: <<>> 76 | ``` 77 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Create user 2 | ansible.builtin.include_tasks: create-user.yml 3 | when: not (restart_only | default(false)) 4 | 5 | - name: Install JVM 6 | ansible.builtin.package: 7 | name: java-17-openjdk 8 | state: present 9 | when: 10 | - not use_existing_jvm | default(false) 11 | - not (restart_only | default(false)) 12 | 13 | - name: Set JAVA_HOME 14 | ansible.builtin.include_tasks: generate-java-home.yml 15 | when: not (restart_only | default(false)) 16 | 17 | - name: Copy truststore 18 | ansible.builtin.include_tasks: copy-truststore.yml 19 | when: 20 | - copy_truststore | default(false) | bool 21 | - not (restart_only | default(false)) 22 | 23 | - name: Install Certs 24 | ansible.builtin.include_tasks: install-certs.yml 25 | when: 26 | - copy_keystore | default(false) | bool 27 | - not (restart_only | default(false)) 28 | 29 | - name: Copy keystore 30 | ansible.builtin.include_tasks: copy-keystore.yml 31 | when: 32 | - copy_keystore | default(false) | bool 33 | - not (restart_only | default(false)) 34 | 35 | - name: Install Kafka Connect 36 | ansible.builtin.include_tasks: install-connect.yml 37 | when: not (restart_only | default(false)) 38 | 39 | - name: Configure Kafka Connect 40 | ansible.builtin.include_tasks: generate-connect-distributed.yml 41 | when: not (restart_only | default(false)) 42 | 43 | - name: Configure Log4j Config File 44 | ansible.builtin.include_tasks: generate-log4j-config.yml 45 | when: not (restart_only | default(false)) 46 | 47 | - name: Generate JMX Exporter Config 48 | ansible.builtin.include_tasks: generate-jmx-exporter-config.yml 49 | when: not (restart_only | default(false)) 50 | 51 | - name: Configure Systemd Unit File 52 | ansible.builtin.include_tasks: generate-unit-file.yml 53 | when: not (restart_only | default(false)) 54 | 55 | - name: Reload systemd daemon if unit file changed 56 | ansible.builtin.systemd_service: 57 | daemon_reload: true 58 | when: 59 | - systemd_unit_result.changed | default(false) 60 | - inventory_hostname in groups[redpanda_connect_group] 61 | - not (restart_only | default(false)) 62 | 63 | - name: Establish whether restart required 64 | ansible.builtin.set_fact: 65 | restart_required: '{{ restart_only | default(false) or ((connect_distributed_config_result.changed | default(false) or log4j_user_config_result.changed | default(false) or log4j_default_config_result.changed | default(false) or systemd_unit_result.changed | default(false)) and (restart_connect_node | default(true) | bool)) }}' 66 | 67 | - name: Ensure Redpanda Connect is enabled and started 68 | ansible.builtin.systemd_service: 69 | name: redpanda-connect 70 | state: started 71 | enabled: true 72 | when: 73 | - inventory_hostname in groups[redpanda_connect_group] 74 | - not (restart_only | default(false)) 75 | 76 | # Safe restart with serialization - one node at a time 77 | - name: Safe Restart 78 | ansible.builtin.include_tasks: safe-restart.yml 79 | with_items: "{{ ansible_play_hosts }}" 80 | loop_control: 81 | loop_var: cluster_host 82 | tags: 83 | - restart 84 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tests/redpanda-connect.service_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import yaml 4 | from jinja2 import Environment, FileSystemLoader 5 | 6 | class TestRedpandaConnectService(unittest.TestCase): 7 | def test_connect_service_template(self): 8 | # Get the absolute path of the current file 9 | current_dir = os.path.dirname(os.path.abspath(__file__)) 10 | 11 | # Construct the path to the templates directory 12 | templates_dir = os.path.join(current_dir, '..', 'templates') 13 | 14 | # Construct the path to the defaults/main.yml file 15 | defaults_file = os.path.join(current_dir, '..', 'defaults', 'main.yml') 16 | 17 | # Load the defaults from the YAML file 18 | with open(defaults_file, 'r') as f: 19 | defaults = yaml.safe_load(f) 20 | 21 | # Create a Jinja2 environment and load the template from file 22 | env = Environment(loader=FileSystemLoader(templates_dir)) 23 | template = env.get_template('redpanda-connect.service.j2') 24 | 25 | # Define the hostvars and groups for rendering the template 26 | hostvars = { 27 | '10.9.8.7': { 28 | 'ansible_host': '10.9.8.7', 29 | 'private_ip': '1.2.3.4', 30 | }, 31 | '10.9.8.6': { 32 | 'ansible_host': '10.9.8.7', 33 | 'private_ip': '1.2.3.4', 34 | }, 35 | '10.9.8.5': { 36 | 'ansible_host': '10.9.8.7', 37 | 'private_ip': '1.2.3.4', 38 | }, 39 | } 40 | groups = { 41 | 'connect': ['10.9.8.7', '10.9.8.6', '10.9.8.5'] 42 | } 43 | inventory_hostname = '10.9.8.7' 44 | 45 | # Render the template with the provided hostvars, groups, and defaults 46 | rendered_template = template.render(hostvars=hostvars, groups=groups, inventory_hostname=inventory_hostname, **defaults) 47 | 48 | # Define the expected values 49 | expected_user = f"User=redpanda" 50 | expected_group = f"Group=redpanda" 51 | expected_plugin_path = f"Environment=\"CONNECT_PLUGIN_PATH=/opt/kafka/plugins\"" 52 | expected_exec_start = f"ExecStart=/opt/kafka/bin/connect-distributed.sh /opt/kafka/config/connect-distributed.properties" 53 | expected_kafka_opts = f"Environment=\"KAFKA_OPTS= -javaagent:/opt/kafka/redpanda-plugins/jmx-exporter/jmx_prometheus_javaagent.jar=9404:/opt/kafka/config/jmx-exporter-config.json\"" 54 | 55 | # Assert the presence of expected lines in the rendered template 56 | self.assertIn(expected_user, rendered_template) 57 | self.assertIn(expected_group, rendered_template) 58 | self.assertIn(expected_plugin_path, rendered_template) 59 | self.assertIn(expected_exec_start, rendered_template) 60 | 61 | # Assert the presence of optional KAFKA_OPTS environment variable if defined 62 | if 'KAFKA_OPTS' in defaults and defaults['KAFKA_OPTS'] != "": 63 | self.assertIn(f"Environment=\"KAFKA_OPTS={defaults['KAFKA_OPTS']}\"", rendered_template) 64 | 65 | print(rendered_template) 66 | 67 | if __name__ == '__main__': 68 | unittest.main() 69 | -------------------------------------------------------------------------------- /roles/redpanda_logging/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Main tasks for redpanda_logging role 3 | 4 | - name: Check if logging is enabled 5 | ansible.builtin.debug: 6 | msg: "Redpanda logging configuration is {{ 'enabled' if redpanda_logging_enabled else 'disabled' }}" 7 | 8 | - name: Configure Redpanda logging 9 | when: redpanda_logging_enabled 10 | block: 11 | - name: Create log directory if needed 12 | ansible.builtin.file: 13 | path: "{{ redpanda_logging_log_file | dirname }}" 14 | state: directory 15 | owner: "{{ redpanda_logging_owner }}" 16 | group: "{{ redpanda_logging_group }}" 17 | mode: "{{ redpanda_logging_dir_mode }}" 18 | become: true 19 | when: (redpanda_logging_log_file | dirname) != '/var/log' 20 | - name: Configure rsyslog for Redpanda 21 | when: redpanda_logging_rsyslog_enabled 22 | block: 23 | - name: Deploy rsyslog configuration 24 | ansible.builtin.template: 25 | src: redpanda-rsyslog.conf.j2 26 | dest: "/etc/rsyslog.d/{{ redpanda_logging_rsyslog_priority }}-redpanda.conf" 27 | owner: root 28 | group: root 29 | mode: '0644' 30 | become: true 31 | notify: restart rsyslog 32 | 33 | - name: Ensure rsyslog service is running 34 | ansible.builtin.systemd: 35 | name: rsyslog 36 | state: started 37 | enabled: true 38 | become: true 39 | 40 | - name: Configure logrotate for Redpanda logs 41 | when: redpanda_logging_logrotate_enabled 42 | block: 43 | - name: Deploy logrotate configuration 44 | ansible.builtin.template: 45 | src: redpanda-logrotate.conf.j2 46 | dest: /etc/logrotate.d/redpanda 47 | owner: root 48 | group: root 49 | mode: '0644' 50 | become: true 51 | 52 | - name: Create initial log file with correct permissions 53 | ansible.builtin.file: 54 | path: "{{ redpanda_logging_log_file }}" 55 | state: touch 56 | owner: "{{ redpanda_logging_owner }}" 57 | group: "{{ redpanda_logging_group }}" 58 | mode: "{{ redpanda_logging_file_mode }}" 59 | modification_time: preserve 60 | access_time: preserve 61 | become: true 62 | 63 | - name: Configure systemd logging (if enabled) 64 | when: redpanda_logging_systemd_enabled 65 | block: 66 | - name: Create systemd drop-in directory for Redpanda service 67 | ansible.builtin.file: 68 | path: "/etc/systemd/system/redpanda.service.d" 69 | state: directory 70 | owner: root 71 | group: root 72 | mode: '0755' 73 | become: true 74 | 75 | - name: Create systemd logging override 76 | ansible.builtin.copy: 77 | content: | 78 | [Service] 79 | StandardOutput=journal 80 | StandardError=journal 81 | SyslogIdentifier={{ redpanda_logging_program }} 82 | dest: "/etc/systemd/system/redpanda.service.d/logging.conf" 83 | owner: root 84 | group: root 85 | mode: '0644' 86 | become: true 87 | notify: reload systemd 88 | 89 | - name: Display logging configuration status 90 | ansible.builtin.debug: 91 | msg: "Redpanda logs will be written to: {{ redpanda_logging_log_file }}" 92 | when: redpanda_logging_enabled -------------------------------------------------------------------------------- /roles/system_setup/tasks/prepare-data-dir.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Prep Data Dir - Set device_info fact 3 | ansible.builtin.set_fact: 4 | device_info: "{{ hostvars[inventory_hostname].ansible_devices }}" 5 | nvme_device_ids: "{{ hostvars[inventory_hostname].ansible_devices.keys() | map('regex_search', 'nvme.*') | select('string') | list }}" 6 | tags: 7 | - prep_data_dir 8 | 9 | - name: Prep Data Dir - Set nvme_devices_for_raid fact 10 | ansible.builtin.set_fact: 11 | nvme_devices_for_raid: '{{ (nvme_devices_for_raid | default([])) + ["/dev/" + item] }}' 12 | loop: "{{ nvme_device_ids }}" 13 | when: device_info[item]["partitions"] | length == 0 14 | tags: 15 | - prep_data_dir 16 | 17 | - name: Prep Data Dir - Set nvme_devices_for_raid fact with default value 18 | ansible.builtin.set_fact: 19 | nvme_devices_for_raid: "{{ (nvme_devices_for_raid | default([])) }}" 20 | tags: 21 | - prep_data_dir 22 | 23 | - name: Prep Data Dir - Set ephemeral_disk fact 24 | ansible.builtin.set_fact: 25 | ephemeral_disk: "{{ (ephemeral_disk | default(false)) }}" 26 | tags: 27 | - prep_data_dir 28 | 29 | # Prepare raid 0 for multiple devices 30 | - name: Prep Data Dir - handle mdadm (raid 0) 31 | tags: 32 | - prep_data_dir 33 | block: 34 | - name: Prep Data Dir - Define mdadm_arrays variable 35 | ansible.builtin.set_fact: 36 | mdadm_arrays: 37 | - name: md0 38 | devices: "{{ nvme_devices_for_raid }}" 39 | community.general.filesystem: xfs 40 | level: 0 41 | mountpoint: /mnt/vectorized 42 | state: present 43 | - name: Prep Data Dir - Run mdadm 44 | ansible.builtin.include_role: 45 | name: mrlesmithjr.mdadm 46 | when: nvme_devices_for_raid|length > 1 47 | 48 | # Prepare single device 49 | - name: Prep Data Dir - mounting file system (single device) 50 | tags: 51 | - prep_data_dir 52 | block: 53 | - name: Prep Data Dir - Create XFS file system 54 | community.general.filesystem: 55 | fstype: xfs 56 | dev: '{{ nvme_devices_for_raid[0] }}' 57 | when: nvme_devices_for_raid|length == 1 58 | - name: Prep Data Dir - Mount NVMe device 59 | ansible.posix.mount: 60 | path: /mnt/vectorized 61 | src: '{{ nvme_devices_for_raid[0] }}' 62 | fstype: xfs 63 | opts: "{{ 'defaults,nofail' if ephemeral_disk else 'defaults'}}" 64 | dump: "{{ 1 if ephemeral_disk else 0}}" 65 | passno: "{{ 2 if ephemeral_disk else 0}}" 66 | state: mounted 67 | when: nvme_devices_for_raid|length == 1 68 | 69 | - name: Prep Data Dir - mounting file system (raid 0) 70 | tags: 71 | - prep_data_dir 72 | block: 73 | - name: Prep Data Dir - Create XFS file system 74 | community.general.filesystem: 75 | fstype: xfs 76 | dev: /dev/md0 77 | when: nvme_devices_for_raid|length > 1 78 | - name: Prep Data Dir - Mount NVMe device 79 | ansible.posix.mount: 80 | path: /mnt/vectorized 81 | src: /dev/md0 82 | fstype: xfs 83 | opts: "{{ 'defaults,nofail' if ephemeral_disk else 'defaults'}}" 84 | dump: "{{ 1 if ephemeral_disk else 0}}" 85 | passno: "{{ 2 if ephemeral_disk else 0}}" 86 | state: mounted 87 | when: nvme_devices_for_raid|length > 1 88 | 89 | - name: Prep Data Dir - Create data path 90 | ansible.builtin.file: 91 | path: "{{ repdanda_mount_dir }}" 92 | state: directory 93 | owner: redpanda 94 | group: redpanda 95 | mode: "0755" 96 | tags: 97 | - prep_data_dir 98 | -------------------------------------------------------------------------------- /roles/redpanda_broker/templates/configs/defaults.j2: -------------------------------------------------------------------------------- 1 | { 2 | "cluster": { 3 | "rpc_server_tcp_recv_buf": 65536, 4 | "enable_rack_awareness": "{{ true if rack is defined and rack != -1 else false }}" 5 | }, 6 | "node": { 7 | "organization": "{{ redpanda_organization }}", 8 | "cluster_id": "{{ redpanda_cluster_id }}", 9 | "redpanda": { 10 | "empty_seed_starts_cluster": false, 11 | "advertised_kafka_api": [ 12 | { 13 | "address": "{{ hostvars[inventory_hostname].advertised_ip }}", 14 | "port": "{{ redpanda_kafka_port }}" 15 | } 16 | ], 17 | "advertised_rpc_api": { 18 | "address": "{{ hostvars[inventory_hostname].private_ip }}", 19 | "port": "{{ redpanda_rpc_port }}" 20 | }, 21 | "data_directory": "{{ redpanda_data_directory }}", 22 | "rpc_server": { 23 | "address": "{{ hostvars[inventory_hostname].private_ip }}", 24 | "port": "{{ redpanda_rpc_port }}" 25 | }, 26 | "kafka_api": [ 27 | { 28 | "address": "{{ hostvars[inventory_hostname].private_ip }}", 29 | "port": "{{ redpanda_kafka_port }}" 30 | } 31 | ], 32 | "admin": [ 33 | { 34 | "address": "{{ hostvars[inventory_hostname].private_ip }}", 35 | "port": "{{ redpanda_admin_api_port }}" 36 | } 37 | ], 38 | {% if rack is defined and rack != '' %}"rack": "{{ rack | default('null') }}",{% endif %} 39 | "seed_servers": [ 40 | {% for host in groups['redpanda'] %} 41 | { 42 | "host": { 43 | "address": "{{ hostvars[host]['private_ip'] }}", 44 | "port": "{{ redpanda_rpc_port }}" 45 | } 46 | }{% if not loop.last %},{% endif %} 47 | {% endfor %} 48 | ] 49 | }, 50 | "rpk": { 51 | "kafka_api": { 52 | "brokers": [ 53 | {% for host in groups['redpanda'] %} 54 | "{{ hostvars[host]['private_ip'] }}:{{ redpanda_kafka_port }}"{% if not loop.last %},{% endif %} 55 | {% endfor %} 56 | ] 57 | }, 58 | "admin_api": { 59 | "addresses": [ 60 | {% for host in groups['redpanda'] %} 61 | "{{ hostvars[host]['private_ip'] }}:{{ redpanda_admin_api_port }}"{% if not loop.last %},{% endif %} 62 | {% endfor %} 63 | ] 64 | }, 65 | "tune_network": {{ enable_tune_network | default(true) | lower }}, 66 | "tune_disk_scheduler": {{ enable_tune_disk_scheduler | default(true) | lower }}, 67 | "tune_disk_nomerges": {{ enable_tune_disk_nomerges | default(true) | lower }}, 68 | "tune_disk_write_cache": {{ enable_tune_disk_write_cache | default(true) | lower }}, 69 | "tune_disk_irq": {{ enable_tune_disk_irq | default(true) | lower }}, 70 | "tune_cpu": {{ enable_tune_cpu | default(true) | lower }}, 71 | "tune_aio_events": {{ enable_tune_aio_events | default(true) | lower }}, 72 | "tune_clocksource": {{ enable_tune_clocksource | default(true) | lower }}, 73 | "tune_swappiness": {{ enable_tune_swappiness | default(true) | lower }}, 74 | "tune_ballast_file": {{ enable_tune_ballast_file | default(true) | lower }} 75 | }, 76 | "pandaproxy": { 77 | "pandaproxy_api": { 78 | "address": "0.0.0.0", 79 | "port": "{{ redpanda_pandaproxy_port }}" 80 | } 81 | }, 82 | "schema_registry": { 83 | "schema_registry_api": [ 84 | { 85 | "address": "0.0.0.0", 86 | "port": "{{ redpanda_schema_registry_port }}" 87 | } 88 | ], 89 | "schema_registry_replication_factor": {{schema_registry_replication_factor | default(1)}} 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tests/tiered-storage_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import yaml 4 | from jinja2 import Environment, FileSystemLoader 5 | 6 | class TestRedpandaTemplate(unittest.TestCase): 7 | def setUp(self): 8 | # Get the absolute path of the current file 9 | self.current_dir = os.path.dirname(os.path.abspath(__file__)) 10 | 11 | # Construct the path to the templates directory 12 | self.templates_dir = os.path.join(self.current_dir, '..', 'templates/configs') 13 | 14 | # Construct the path to the defaults/main.yml file 15 | self.defaults_file = os.path.join(self.current_dir, '..', 'defaults', 'main.yml') 16 | 17 | # Load the defaults from the YAML file 18 | with open(self.defaults_file, 'r') as f: 19 | self.defaults = yaml.safe_load(f) 20 | 21 | # Create a Jinja2 environment 22 | self.env = Environment(loader=FileSystemLoader(self.templates_dir)) 23 | 24 | # Define the hostvars and groups for rendering the template 25 | self.hostvars = { 26 | '35.91.106.231': { 27 | 'ansible_host': '35.91.106.231', 28 | 'advertised_ip': '35.91.106.231', 29 | 'private_ip': '192.168.1.1' 30 | }, 31 | '35.88.129.205': { 32 | 'ansible_host': '35.88.129.205', 33 | 'advertised_ip': '35.88.129.205', 34 | 'private_ip': '192.168.1.2' 35 | }, 36 | '54.190.184.126': { 37 | 'ansible_host': '54.190.184.126', 38 | 'advertised_ip': '54.190.184.126', 39 | 'private_ip': '192.168.1.3' 40 | } 41 | } 42 | self.groups = { 43 | 'redpanda': ['35.91.106.231', '35.88.129.205', '54.190.184.126'] 44 | } 45 | 46 | def test_cluster_template_cloud_storage(self): 47 | # Load the template from file 48 | template = self.env.get_template('tiered_storage.j2') 49 | 50 | # Define the variables for the template 51 | context = { 52 | 'tiered_storage_bucket_name': 'my_bucket', 53 | 'cloud_storage_enable_remote_read': 'True', 54 | 'cloud_storage_enable_remote_write': 'True', 55 | 'cloud_storage_region': 'us-west-2', 56 | 'cloud_storage_credentials_source': 'gcp_instance_metadata' 57 | } 58 | 59 | # Render the template with the provided variables, hostvars, and groups 60 | rendered_template = template.render( 61 | hostvars=self.hostvars, 62 | groups=self.groups, 63 | inventory_hostname='35.91.106.231', 64 | **context 65 | ) 66 | 67 | # Define the expected rendered values 68 | expected_rendered_values = { 69 | "cluster": { 70 | "cloud_storage_bucket": "my_bucket", 71 | "cloud_storage_enable_remote_read": 'True', 72 | "cloud_storage_enable_remote_write": 'True', 73 | "cloud_storage_region": "us-west-2", 74 | "cloud_storage_credentials_source": "gcp_instance_metadata", 75 | "cloud_storage_enabled": 'True', 76 | "cloud_storage_api_endpoint": "storage.googleapis.com" 77 | } 78 | } 79 | 80 | # Convert the rendered template to a dictionary for easier comparison 81 | rendered_dict = yaml.safe_load(rendered_template) 82 | 83 | # Assert that the expected values are in the rendered template 84 | self.assertEqual(rendered_dict, expected_rendered_values) 85 | 86 | print(rendered_template) 87 | 88 | if __name__ == '__main__': 89 | unittest.main() 90 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tests/restart_required_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | import ansible_runner 4 | import json 5 | def run_playbook(extra_vars): 6 | playbook_path = '/app/tests/restart_required.yml' 7 | inventory_path = '/app/tests/inventory' 8 | 9 | # Ensure the inventory file exists and contains localhost 10 | with open(inventory_path, 'w') as f: 11 | f.write('localhost ansible_connection=local') 12 | 13 | # Add mock shell command values to extra_vars 14 | extra_vars['mock_shell_stdout'] = extra_vars['restart_required_rc']['stdout'] 15 | extra_vars['mock_shell_rc'] = 0 if extra_vars['restart_required_rc']['stdout'] in ['true', 'false'] else 1 16 | 17 | # Run the playbook 18 | r = ansible_runner.run( 19 | playbook=playbook_path, 20 | inventory=inventory_path, 21 | extravars=extra_vars, 22 | quiet=False 23 | ) 24 | 25 | # Check if the playbook run was successful 26 | if r.status != 'successful': 27 | print(f"Playbook stdout:\n{r.stdout.read()}") 28 | print(f"Playbook stderr:\n{r.stderr.read()}") 29 | for event in r.events: 30 | if event['event'] == 'runner_on_failed': 31 | print(f"Task failed: {event['event_data']['task']}") 32 | print(f"Error message: {event['event_data']['res'].get('msg', 'No error message')}") 33 | assert False, f"Playbook failed: {r.rc}" 34 | 35 | # Parse the output to get the restart_required value 36 | for event in r.events: 37 | if event['event'] == 'runner_on_ok' and 'restart_required' in event['event_data']['res'].get('ansible_facts', {}): 38 | return event['event_data']['res']['ansible_facts']['restart_required'] 39 | 40 | raise ValueError("Could not find restart_required in playbook output") 41 | 42 | @pytest.mark.parametrize("test_input,expected", [ 43 | ( 44 | { 45 | "is_initialized": False, 46 | "nodeconfig_result": {"changed": False}, 47 | "package_result": {"results": []}, 48 | "restart_node": True, 49 | "restart_required_rc": {"stdout": "false"}, 50 | }, 51 | False 52 | ), 53 | ( 54 | { 55 | "is_initialized": False, 56 | "nodeconfig_result": {"changed": False}, 57 | "package_result": {"results": []}, 58 | "restart_node": True, 59 | "restart_required_rc": {"stdout": "true"} 60 | }, 61 | True 62 | ), 63 | ( 64 | { 65 | "is_initialized": True, 66 | "nodeconfig_result": {"changed": True}, 67 | "package_result": {"results": []}, 68 | "restart_node": True, 69 | "restart_required_rc": {"stdout": "false"}, 70 | "mock_template_changed": False 71 | }, 72 | True 73 | ), 74 | ( 75 | { 76 | "is_initialized": True, 77 | "nodeconfig_result": {"changed": False}, 78 | "package_result": {"results": ["1 upgraded"]}, 79 | "restart_node": True, 80 | "restart_required_rc": {"stdout": "false"}, 81 | "mock_template_changed": False 82 | }, 83 | True 84 | ), 85 | ( 86 | { 87 | "is_initialized": True, 88 | "nodeconfig_result": {"changed": False}, 89 | "package_result": {"results": []}, 90 | "restart_node": True, 91 | "restart_required_rc": {"stdout": "true"}, 92 | "mock_template_changed": True 93 | }, 94 | True 95 | ), 96 | ( 97 | { 98 | "is_initialized": True, 99 | "nodeconfig_result": {"changed": True}, 100 | "package_result": {"results": ["1 upgraded"]}, 101 | "restart_node": False, 102 | "restart_required_rc": {"stdout": "true"}, 103 | "mock_template_changed": False 104 | }, 105 | False 106 | ), 107 | ]) 108 | def test_restart_required(test_input, expected): 109 | restart_required = run_playbook(test_input) 110 | assert restart_required == expected, f"Expected restart_required to be {expected}, but got {restart_required}" 111 | 112 | if __name__ == "__main__": 113 | pytest.main([__file__, "-v"]) 114 | -------------------------------------------------------------------------------- /roles/redpanda_connect/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # connect distributed config file - allows overriding the config file 2 | connect_distributed_config_file: connect-distributed.properties 3 | 4 | # Kafka Cluster connection values 5 | is_using_private: false # set to true if you'd prefer it connected using the private IP 6 | kafka_port: 9092 7 | redpanda_connect_home: "/opt/kafka" 8 | 9 | # converters 10 | key_converter: "org.apache.kafka.connect.json.JsonConverter" 11 | value_converter: "org.apache.kafka.connect.json.JsonConverter" 12 | 13 | # connect topics 14 | config_storage_topic: "connect-configs" 15 | offset_storage_topic: "connect-offsets" 16 | status_storage_topic: "connect-status" 17 | 18 | # config directory 19 | redpanda_connect_config_dir: "/opt/kafka/config" 20 | 21 | # user and group 22 | redpanda_user: redpanda 23 | redpanda_group: redpanda 24 | 25 | java_home: /usr/lib/jvm/java-17-openjdk 26 | 27 | # required binary install values 28 | redpanda_connect_rpm: "redpanda-connect.x86_64.rpm" 29 | redpanda_connect_rpm_dir: "/tmp" 30 | 31 | # networking 32 | advertise_public_address: true 33 | rest_port: 8083 34 | rest_advertised_port: 8083 35 | 36 | # group and plugins 37 | group_id: "redpanda-kc" 38 | plugin_path: "/opt/kafka/redpanda-plugins" 39 | plugin_discovery: hybrid_warn 40 | 41 | # security 42 | connect_tls_enabled: false 43 | connect_trusted_certs: false 44 | connect_tls_auth_cert: false 45 | connect_tls_auth_key: false 46 | 47 | # used when connect_tls_enabled are true 48 | security_protocol: SSL 49 | ssl_protocol: TLSv1.3 50 | 51 | # jmx options 52 | prefer_ipv4_stack: true 53 | jmx_keystore_path: /etc/redpanda/keystores/keystore.p12 54 | jmx_keystore_password: password 55 | jmx_truststore_path: /etc/redpanda/truststores/truststore.p12 56 | jmx_truststore_password: password 57 | jmx_certificate_alias: localhost 58 | kafka_additional_opts: "" 59 | 60 | # used when connect_tls_enabled and connect_trusted_certs are true 61 | redpanda_truststores_dir: /etc/redpanda/truststores 62 | truststore_file_name: truststore.p12 63 | ssl_truststore_location: /etc/redpanda/truststores/truststore.p12 64 | ssl_truststore_password: password 65 | ssl_truststore_type: PKCS12 66 | producer_ssl_truststore_location: /etc/redpanda/truststores/truststore.p12 67 | producer_ssl_truststore_password: password 68 | consumer_ssl_truststore_location: /etc/redpanda/truststores/truststore.p12 69 | consumer_ssl_truststore_password: password 70 | admin_ssl_truststore_location: /etc/redpanda/truststores/truststore.p12 71 | admin_ssl_truststore_password: password 72 | 73 | # use when connect_tls_enabled, connect_tls_auth_cert and connect_tls_auth_key are true 74 | redpanda_keystores_dir: /etc/redpanda/keystores 75 | keystores_file_name: keystore.p12 76 | ssl_keystore_location: /etc/redpanda/keystores/keystore.p12 77 | ssl_keystore_password: password 78 | ssl_keystore_type: PKCS12 79 | producer_ssl_keystore_location: /etc/redpanda/keystores/keystore.p12 80 | producer_ssl_keystore_password: password 81 | producer_ssl_keystore_type: PKCS12 82 | consumer_ssl_keystore_location: /etc/redpanda/keystores/keystore.p12 83 | consumer_ssl_keystore_password: password 84 | consumer_ssl_keystore_type: PKCS12 85 | admin_ssl_keystore_location: /etc/redpanda/keystores/keystore.p12 86 | admin_ssl_keystore_password: password 87 | admin_ssl_keystore_type: PKCS12 88 | 89 | # log4j values 90 | kafka_log4j_opts: "-Dlog4j.configuration=file:{{ redpanda_connect_config_dir }}/connect-log4j.properties -Dlog4j_log_level=WARN" 91 | log4j_log_level: "WARN" 92 | 93 | # kafka startup options 94 | kafka_opts: "-javaagent:{{ redpanda_connect_home }}/redpanda-plugins/jmx-exporter/jmx_prometheus_javaagent.jar=9404:{{ redpanda_connect_home }}/config/jmx-exporter-config.json" 95 | 96 | # used for keystore gen 97 | redpanda_certs_dir: /etc/redpanda/certs 98 | redpanda_csr_file: "{{ redpanda_certs_dir }}/node.csr" 99 | redpanda_key_file: "{{ redpanda_certs_dir }}/node.key" 100 | redpanda_cert_file: "{{ redpanda_certs_dir }}/node.crt" 101 | redpanda_truststore_file: "{{ redpanda_certs_dir }}/truststore.pem" 102 | 103 | timeout_start_sec: 30 104 | 105 | # ansible group name for connect hosts 106 | redpanda_connect_group: "connect" 107 | 108 | # restart control - set to false to prevent automatic restarts 109 | restart_connect_node: true 110 | 111 | # restart-only mode - set to true to skip installation/config and only restart 112 | restart_only: false 113 | 114 | # health check settings for safe rolling restarts 115 | connect_restart_health_check_timeout: 120 # total seconds to wait for API 116 | connect_restart_health_check_retries: 24 # number of health check attempts 117 | connect_restart_health_check_delay: 5 # seconds between retries 118 | -------------------------------------------------------------------------------- /roles/redpanda_connect/tests/connect-distributed.properties_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import yaml 4 | from jinja2 import Environment, FileSystemLoader 5 | 6 | class TestConnectDistributedProperties(unittest.TestCase): 7 | def test_bootstrap_server_template(self): 8 | # Get the absolute path of the current file 9 | current_dir = os.path.dirname(os.path.abspath(__file__)) 10 | 11 | # Construct the path to the templates directory 12 | templates_dir = os.path.join(current_dir, '..', 'templates') 13 | 14 | # Construct the path to the defaults/main.yml file 15 | defaults_file = os.path.join(current_dir, '..', 'defaults', 'main.yml') 16 | 17 | # Load the defaults from the YAML file 18 | with open(defaults_file, 'r') as f: 19 | defaults = yaml.safe_load(f) 20 | 21 | # Create a Jinja2 environment and load the template from file 22 | env = Environment(loader=FileSystemLoader(os.path.join(templates_dir, 'connect-distributed'))) 23 | template = env.get_template('connect-distributed.properties.j2') 24 | 25 | # Define the hostvars and groups for rendering the template 26 | hostvars = { 27 | '35.91.106.231': { 28 | 'ansible_host': '35.91.106.231', 29 | }, 30 | '35.88.129.205': { 31 | 'ansible_host': '35.88.129.205', 32 | }, 33 | '54.190.184.126': { 34 | 'ansible_host': '54.190.184.126', 35 | } 36 | } 37 | advertised_ips = ["35.91.106.231", "35.88.129.205", "54.190.184.126"] 38 | groups = { 39 | 'connect': ['35.91.106.231', '35.88.129.205', '54.190.184.126'], 40 | 'redpanda': ['35.91.106.231'] # Add redpanda group with first node 41 | } 42 | 43 | # Add missing variables that are calculated in Ansible 44 | connect_tls_enabled = defaults.get('connect_tls_enabled', False) 45 | schema_registry_actual = defaults.get('schema_registry_url', 46 | f"http{'s' if connect_tls_enabled else ''}://{groups['redpanda'][0]}:8081") 47 | defaults['schema_registry_actual'] = schema_registry_actual 48 | 49 | # Set additional missing values with dummy data if they don't exist in defaults 50 | if 'config_storage_topic' not in defaults: 51 | defaults['config_storage_topic'] = 'connect-configs' 52 | if 'offset_storage_topic' not in defaults: 53 | defaults['offset_storage_topic'] = 'connect-offsets' 54 | if 'status_storage_topic' not in defaults: 55 | defaults['status_storage_topic'] = 'connect-status' 56 | if 'plugin_discovery' not in defaults: 57 | defaults['plugin_discovery'] = 'org.apache.kafka.connect.discovery.PluginDiscovery' 58 | if 'key_converter' not in defaults: 59 | defaults['key_converter'] = 'org.apache.kafka.connect.json.JsonConverter' 60 | if 'value_converter' not in defaults: 61 | defaults['value_converter'] = 'org.apache.kafka.connect.json.JsonConverter' 62 | if 'kafka_port' not in defaults: 63 | defaults['kafka_port'] = 9092 64 | if 'rest_advertised_host_name_actual' not in defaults: 65 | defaults['rest_advertised_host_name_actual'] = defaults.get('rest_advertised_host_name', 'localhost') 66 | 67 | # Render the template with the provided hostvars, groups, and defaults 68 | rendered_template = template.render(hostvars=hostvars, advertised_ips=advertised_ips, groups=groups, **defaults) 69 | 70 | # Define the expected bootstrap server line 71 | expected_bootstrap_servers = 'bootstrap.servers=35.91.106.231:9092,35.88.129.205:9092,54.190.184.126:9092' 72 | expected_schema_registry_url = f"schema.registry.url={schema_registry_actual}" 73 | expected_group_id = f"group.id={defaults['group_id']}" 74 | expected_rest_port = f"rest.port={defaults['rest_port']}" 75 | expected_rest_advertised_host_name = f"rest.advertised.host.name={defaults['rest_advertised_host_name_actual']}" 76 | expected_rest_advertised_port = f"rest.advertised.port={defaults['rest_advertised_port']}" 77 | expected_plugin_path = f"plugin.path={defaults['plugin_path']}" 78 | 79 | # Assert that the expected line is contained within the rendered template 80 | self.assertIn(expected_bootstrap_servers, rendered_template) 81 | self.assertIn(expected_schema_registry_url, rendered_template) 82 | self.assertIn(expected_group_id, rendered_template) 83 | self.assertIn(expected_rest_port, rendered_template) 84 | self.assertIn(expected_rest_advertised_host_name, rendered_template) 85 | self.assertIn(expected_rest_advertised_port, rendered_template) 86 | self.assertIn(expected_plugin_path, rendered_template) 87 | 88 | print(rendered_template) 89 | 90 | if __name__ == '__main__': 91 | unittest.main() 92 | -------------------------------------------------------------------------------- /roles/redpanda_logging/README.md: -------------------------------------------------------------------------------- 1 | # Redpanda Logging Role 2 | 3 | This Ansible role configures centralized logging for Redpanda to a dedicated log file, preventing system log pollution and providing better log management capabilities. 4 | 5 | ## Overview 6 | 7 | The role addresses the issue of Redpanda logs mixing with system logs by: 8 | - Redirecting logs to `/var/log/redpanda.log` 9 | - Setting up log rotation to prevent disk space issues 10 | - Configuring proper permissions and ownership 11 | - Supporting both rsyslog and systemd logging configurations 12 | 13 | ## Requirements 14 | 15 | - Ansible 2.9 or higher 16 | - Target systems running systemd 17 | - rsyslog installed on target systems (for rsyslog-based logging) 18 | - logrotate installed on target systems (for log rotation) 19 | 20 | ## Role Variables 21 | 22 | ### Basic Configuration 23 | 24 | | Variable | Default | Description | 25 | |----------|---------|-------------| 26 | | `redpanda_logging_enabled` | `true` | Enable/disable logging configuration | 27 | | `redpanda_logging_log_file` | `/var/log/redpanda.log` | Redpanda log file path | 28 | 29 | ### Permissions 30 | 31 | | Variable | Default | Description | 32 | |----------|---------|-------------| 33 | | `redpanda_logging_owner` | `syslog` | Log file owner | 34 | | `redpanda_logging_group` | `adm` | Log file group | 35 | | `redpanda_logging_dir_mode` | `0755` | Log directory permissions | 36 | | `redpanda_logging_file_mode` | `0640` | Log file permissions | 37 | 38 | ### Rsyslog Configuration 39 | 40 | | Variable | Default | Description | 41 | |----------|---------|-------------| 42 | | `redpanda_logging_rsyslog_enabled` | `true` | Enable rsyslog configuration | 43 | | `redpanda_logging_rsyslog_priority` | `40` | Rsyslog config file priority | 44 | | `redpanda_logging_program` | `rpk` | Syslog program name for Redpanda | 45 | 46 | ### Logrotate Configuration 47 | 48 | | Variable | Default | Description | 49 | |----------|---------|-------------| 50 | | `redpanda_logging_logrotate_enabled` | `true` | Enable logrotate configuration | 51 | | `redpanda_logging_logrotate_frequency` | `daily` | Log rotation frequency | 52 | | `redpanda_logging_logrotate_rotate` | `7` | Number of rotated logs to keep | 53 | | `redpanda_logging_logrotate_maxsize` | `100M` | Maximum log file size before rotation | 54 | | `redpanda_logging_logrotate_compress` | `true` | Compress rotated logs | 55 | | `redpanda_logging_logrotate_delaycompress` | `true` | Delay compression until next rotation | 56 | | `redpanda_logging_logrotate_notifempty` | `true` | Don't rotate empty logs | 57 | 58 | ### Systemd Configuration 59 | 60 | | Variable | Default | Description | 61 | |----------|---------|-------------| 62 | | `redpanda_logging_systemd_enabled` | `false` | Enable systemd-specific logging config | 63 | | `redpanda_logging_systemd_max_level` | `info` | Maximum log level for journald | 64 | | `redpanda_logging_systemd_forward_to_syslog` | `true` | Forward systemd logs to syslog | 65 | 66 | ## Dependencies 67 | 68 | None. This role is designed to work independently. 69 | 70 | ## Example Playbook 71 | 72 | ### Basic Usage 73 | 74 | ```yaml 75 | - hosts: redpanda_nodes 76 | roles: 77 | - role: redpanda_logging 78 | ``` 79 | 80 | ### Custom Configuration 81 | 82 | ```yaml 83 | - hosts: redpanda_nodes 84 | roles: 85 | - role: redpanda_logging 86 | vars: 87 | redpanda_logging_log_file: /data/logs/redpanda.log 88 | redpanda_logging_logrotate_rotate: 14 89 | redpanda_logging_logrotate_maxsize: 500M 90 | ``` 91 | 92 | ### Enterprise Configuration with Custom Mount Point 93 | 94 | ```yaml 95 | - hosts: redpanda_nodes 96 | roles: 97 | - role: redpanda_logging 98 | vars: 99 | # Use custom log file path 100 | redpanda_logging_log_file: /app/logs/redpanda.log 101 | 102 | # Keep more logs for compliance 103 | redpanda_logging_logrotate_rotate: 30 104 | redpanda_logging_logrotate_maxsize: 1G 105 | 106 | # Custom ownership if needed 107 | redpanda_logging_owner: redpanda 108 | redpanda_logging_group: redpanda 109 | ``` 110 | 111 | ## How It Works 112 | 113 | 1. **Directory Creation**: Creates the log directory structure with proper permissions 114 | 2. **Rsyslog Configuration**: Deploys rsyslog rules to filter Redpanda logs by program name 115 | 3. **Log File Creation**: Creates initial log files with correct ownership 116 | 4. **Logrotate Setup**: Configures automatic log rotation to prevent disk space issues 117 | 5. **Service Integration**: Optionally configures systemd service overrides for consistent logging 118 | 119 | ## Troubleshooting 120 | 121 | ### Logs Not Appearing 122 | 123 | 1. Check rsyslog is running: `systemctl status rsyslog` 124 | 2. Verify rsyslog configuration: `rsyslogd -N1` 125 | 3. Check service is using correct syslog identifier: `journalctl -u redpanda -o json | grep SYSLOG_IDENTIFIER` 126 | 127 | ### Permission Issues 128 | 129 | 1. Verify log file ownership: `ls -la /var/log/redpanda.log` 130 | 2. Check SELinux context if enabled: `ls -Z /var/log/redpanda.log` 131 | 3. Ensure rsyslog can write to log file: `sudo -u syslog touch /var/log/redpanda.log` 132 | 133 | ### Log Rotation Not Working 134 | 135 | 1. Test logrotate configuration: `logrotate -d /etc/logrotate.d/redpanda` 136 | 2. Force rotation: `logrotate -f /etc/logrotate.d/redpanda` 137 | 3. Check logrotate status: `cat /var/lib/logrotate/status` 138 | 139 | ## License 140 | 141 | Same as redpanda-ansible-collection 142 | 143 | ## Author Information 144 | 145 | Created for the redpanda-ansible-collection to address enterprise logging requirements. -------------------------------------------------------------------------------- /roles/redpanda_console/tests/defaults_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import yaml 4 | from jinja2 import Environment, FileSystemLoader 5 | import json 6 | 7 | 8 | class TestDefaultsTemplate(unittest.TestCase): 9 | def test_defaults_template(self): 10 | # Get the absolute path of the current file 11 | current_dir = os.path.dirname(os.path.abspath(__file__)) 12 | 13 | # Construct the path to the templates directory 14 | templates_dir = os.path.join(current_dir, '..', 'templates') 15 | 16 | # Construct the path to the defaults/main.yml file 17 | defaults_file = os.path.join(current_dir, '..', 'defaults', 'main.yml') 18 | 19 | # Load the defaults from the YAML file 20 | with open(defaults_file, 'r') as f: 21 | defaults = yaml.safe_load(f) 22 | 23 | # Create a Jinja2 environment and load the template from file 24 | env = Environment(loader=FileSystemLoader(templates_dir)) 25 | template = env.get_template('defaults.j2') 26 | 27 | # Define the hostvars and groups for rendering the template 28 | hostvars = { 29 | '35.91.106.231': { 30 | 'ansible_host': '35.91.106.231', 31 | }, 32 | '35.88.129.205': { 33 | 'ansible_host': '35.88.129.205', 34 | }, 35 | '54.190.184.126': { 36 | 'ansible_host': '54.190.184.126', 37 | } 38 | } 39 | advertised_ips = ["35.91.106.231", "35.88.129.205", "54.190.184.126"] 40 | groups = { 41 | 'redpanda': ['35.91.106.231', '35.88.129.205', '54.190.184.126'] 42 | } 43 | 44 | # Set up some test cases for enable_tls 45 | scenarios = [ 46 | {'enable_tls': False, 'name': 'without TLS'}, 47 | {'enable_tls': True, 'name': 'with TLS'} 48 | ] 49 | 50 | for scenario in scenarios: 51 | # Merge scenario settings with defaults 52 | test_vars = {**defaults, **scenario} 53 | 54 | # Render the template 55 | rendered_template = template.render( 56 | hostvars=hostvars, 57 | advertised_ips=advertised_ips, 58 | groups=groups, 59 | **test_vars 60 | ) 61 | 62 | # Parse the rendered JSON to validate its structure 63 | try: 64 | parsed_json = json.loads(rendered_template) 65 | 66 | # Test that expected sections exist in the JSON 67 | self.assertIn("kafka", parsed_json) 68 | self.assertIn("brokers", parsed_json["kafka"]) 69 | self.assertIn("redpanda", parsed_json) 70 | self.assertIn("adminApi", parsed_json["redpanda"]) 71 | self.assertIn("schemaRegistry", parsed_json) 72 | 73 | # Test that TLS settings are present when enable_tls is True 74 | if scenario['enable_tls']: 75 | self.assertIn("tls", parsed_json["kafka"]) 76 | self.assertTrue(parsed_json["kafka"]["tls"]["enabled"]) 77 | self.assertIn("tls", parsed_json["redpanda"]["adminApi"]) 78 | self.assertTrue(parsed_json["redpanda"]["adminApi"]["tls"]["enabled"]) 79 | self.assertIn("server", parsed_json) 80 | self.assertIn("tls", parsed_json["server"]) 81 | self.assertTrue(parsed_json["server"]["tls"]["enabled"]) 82 | else: 83 | self.assertNotIn("tls", parsed_json.get("kafka", {})) 84 | 85 | # Test kafka brokers 86 | self.assertEqual(len(parsed_json["kafka"]["brokers"]), 3) 87 | for i, ip in enumerate(advertised_ips): 88 | expected_broker = f"{ip}:{defaults['redpanda_kafka_port']}" 89 | self.assertEqual(parsed_json["kafka"]["brokers"][i], expected_broker) 90 | 91 | # Test admin API URLs 92 | admin_urls = parsed_json["redpanda"]["adminApi"]["urls"] 93 | self.assertEqual(len(admin_urls), 3) 94 | protocol = "https" if scenario['enable_tls'] else "http" 95 | for i, ip in enumerate(advertised_ips): 96 | expected_url = f"{protocol}://{ip}:{defaults['redpanda_admin_api_port']}" 97 | self.assertEqual(admin_urls[i], expected_url) 98 | 99 | # Test schema registry URLs 100 | schema_urls = parsed_json["schemaRegistry"]["urls"] 101 | self.assertEqual(len(schema_urls), 3) 102 | for i, ip in enumerate(advertised_ips): 103 | expected_url = f"{protocol}://{ip}:{defaults['redpanda_schema_registry_port']}" 104 | self.assertEqual(schema_urls[i], expected_url) 105 | 106 | # Test kafkaConnect settings if applicable 107 | if "connect" in groups: 108 | self.assertIn("kafkaConnect", parsed_json) 109 | self.assertTrue(parsed_json["kafkaConnect"]["enabled"]) 110 | self.assertEqual(len(parsed_json["kafkaConnect"]["clusters"]), 1) 111 | self.assertEqual(parsed_json["kafkaConnect"]["clusters"][0]["name"], "Connect") 112 | 113 | print(f"Successfully validated scenario: {scenario['name']}") 114 | 115 | except json.JSONDecodeError as e: 116 | self.fail(f"Failed to parse rendered JSON template: {e}") 117 | print(rendered_template) # Print the template to debug 118 | 119 | # Print the rendered template for debugging 120 | print(rendered_template) 121 | 122 | 123 | if __name__ == '__main__': 124 | unittest.main() 125 | -------------------------------------------------------------------------------- /roles/redpanda_broker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | redpanda_organization: redpanda-test 3 | redpanda_cluster_id: redpanda 4 | enable_airgap: false 5 | 6 | redpanda_mode: production 7 | redpanda_admin_api_port: 9644 8 | redpanda_kafka_port: 9092 9 | redpanda_rpc_port: 33145 10 | redpanda_schema_registry_port: 8081 11 | redpanda_pandaproxy_port: 8082 12 | require_client_auth: false 13 | redpanda_user: redpanda 14 | redpanda_group: redpanda 15 | 16 | redpanda_use_staging_repo: false 17 | redpanda_base_url: "https://dl.redpanda.com" 18 | 19 | redpanda_install_status: present # If redpanda_version is set to latest, changing redpanda_install_status to latest will effect an upgrade, otherwise the currently installed version will remain 20 | redpanda_rpk_opts: "" 21 | 22 | redpanda_base_dir: /var/lib/redpanda 23 | redpanda_data_directory: "{{ redpanda_base_dir }}/data" 24 | repdanda_mount_dir: /mnt/vectorized/redpanda 25 | 26 | redpanda_certs_dir: /etc/redpanda/certs 27 | redpanda_csr_file: "{{ redpanda_certs_dir }}/node.csr" 28 | redpanda_key_file: "{{ redpanda_certs_dir }}/node.key" 29 | redpanda_cert_file: "{{ redpanda_certs_dir }}/node.crt" 30 | redpanda_truststore_file: "{{ redpanda_certs_dir }}/truststore.pem" 31 | node_exporter_version: "{{ node_exporter_custom_version | default('1.5.0') }}" 32 | cloud_storage_credentials_source: "config_file" 33 | cloud_storage_enable_remote_write: "true" 34 | cloud_storage_enable_remote_read: "true" 35 | is_using_unstable: false 36 | 37 | # We default to use the same redpanda dirs, but in prod, a user can specify them directly 38 | console_certs_dir: "{{ redpanda_certs_dir }}" 39 | console_key_file: "{{ console_certs_dir }}/node.key" 40 | console_cert_file: "{{ console_certs_dir }}/node.crt" 41 | 42 | airgap_copy_src: "/tmp" 43 | airgap_copy_dest: "/tmp" 44 | 45 | # the rpms 46 | rp_key_rpm: "{{ redpanda_base_url }}/public/redpanda/gpg.988A7B0A4918BC85.key" 47 | rp_key_rpm_unstable: "{{ redpanda_base_url }}/E4xN1tVe3Xy60GTx/redpanda-unstable/gpg.39C20393EC2E8747.key" 48 | rp_standard_rpm: "{{ redpanda_base_url }}/public/redpanda/rpm/{{ rp_rpm_distro_map[ansible_distribution] | default('el') }}/{{ ansible_distribution_major_version }}/$basearch" 49 | rp_standard_rpm_unstable: "{{ redpanda_base_url }}/E4xN1tVe3Xy60GTx/redpanda-unstable/rpm/{{ rp_rpm_distro_map[ansible_distribution] | default('el') }}/{{ ansible_distribution_major_version }}/$basearch" 50 | rp_noarch_rpm: "{{ redpanda_base_url }}/public/redpanda/rpm/{{ rp_rpm_distro_map[ansible_distribution] | default('el') }}/{{ ansible_distribution_major_version }}/noarch" 51 | rp_noarch_rpm_unstable: "{{ redpanda_base_url }}/E4xN1tVe3Xy60GTx/redpanda-unstable/rpm/{{ rp_rpm_distro_map[ansible_distribution] | default('el') }}/{{ ansible_distribution_major_version }}/noarch" 52 | rp_source_rpm: "{{ redpanda_base_url }}/public/redpanda/rpm/{{ rp_rpm_distro_map[ansible_distribution] | default('el') }}/{{ ansible_distribution_major_version }}/SRPMS" 53 | rp_source_rpm_unstable: "{{ redpanda_base_url }}/E4xN1tVe3Xy60GTx/redpanda-unstable/rpm/{{ rp_rpm_distro_map[ansible_distribution] | default('el') }}/{{ ansible_distribution_major_version }}/SRPMS" 54 | rp_conf_loc_rpm: "/etc/yum.repos.d/redpanda-redpanda.repo" 55 | rp_rpm_distro_map: 56 | Fedora: fedora 57 | Amazon: amzn 58 | default: el 59 | 60 | 61 | # the debs 62 | rp_key_deb: "{{ redpanda_base_url }}/sMIXnoa7DK12JW4A/redpanda/gpg.988A7B0A4918BC85.key" 63 | rp_key_deb_unstable: "{{ redpanda_base_url }}/E4xN1tVe3Xy60GTx/redpanda-unstable/gpg.39C20393EC2E8747.key" 64 | rp_conf_loc_deb: "/etc/apt/sources.list.d/redpanda.list" 65 | rp_conf_loc_deb_unstable: "etc/apt/sources.list.d/redpanda-redpanda-unstable.list" 66 | rp_key_path_deb: "/usr/share/keyrings/redpanda-redpanda-archive-keyring.gpg" 67 | rp_key_path_deb_unstable: "/usr/share/keyrings/redpanda-redpanda-unstable-archive-keyring.gpg" 68 | rp_repo_signing_deb: "deb [signed-by=/usr/share/keyrings/redpanda-redpanda-archive-keyring.gpg] {{ redpanda_base_url }}/public/redpanda/deb/{{ansible_distribution | lower}} {{ ansible_distribution_release | lower }} main" 69 | rp_repo_signing_deb_unstable: "deb [signed-by=/usr/share/keyrings/redpanda-redpanda-unstable-archive-keyring.gpg] {{ redpanda_base_url }}/E4xN1tVe3Xy60GTx/redpanda-unstable/deb/{{ansible_distribution | lower}} {{ ansible_distribution_release | lower }} main" 70 | rp_repo_signing_src_deb: "deb-src [signed-by=/usr/share/keyrings/redpanda-redpanda-archive-keyring.gpg] {{ redpanda_base_url }}/public/redpanda/deb/{{ansible_distribution | lower}} {{ ansible_distribution_release | lower }} main" 71 | rp_repo_signing_src_deb_unstable: "deb-src [signed-by=/usr/share/keyrings/redpanda-redpanda-unstable-archive-keyring.gpg] {{ redpanda_base_url }}/E4xN1tVe3Xy60GTx/redpanda-unstable/deb/{{ansible_distribution | lower}} {{ ansible_distribution_release | lower }} main" 72 | 73 | redpanda_broker_no_log: true 74 | 75 | # used for installing the nightly build 76 | cloudsmith_token: "{{ lookup('env', 'CLOUDSMITH_API_KEY') }}" 77 | cloudsmith_gpg_key_url: "https://dl.redpanda.com/{{ cloudsmith_token }}/redpanda-nightly/gpg.E2242C4583FC4E52.key" 78 | cloudsmith_config_url_deb: "https://dl.redpanda.com/{{ cloudsmith_token }}/redpanda-nightly/config.deb.txt" 79 | cloudsmith_config_url_rpm: "https://dl.redpanda.com/{{ cloudsmith_token }}/redpanda-nightly/config.rpm.txt" 80 | keyring_location: '/usr/share/keyrings/redpanda-redpanda-nightly-archive-keyring.gpg' 81 | development_build: false 82 | 83 | 84 | 85 | # enable fips. requires an enterprise license. 86 | # see https://docs.redpanda.com/current/manage/security/fips-compliance/ 87 | enable_fips: false 88 | fips_mode: disabled 89 | fips_openssl_config_file: "/opt/redpanda/openssl/openssl.cnf" 90 | fips_openssl_module_directory: "/opt/redpanda/lib/ossl-modules/" 91 | 92 | 93 | # fips vars for checking status 94 | fips_check_exit_codes: 95 | - 0 # fips-mode-setup is enabled 96 | - 1 # fips-mode-setup is inconsistent 97 | - 2 # fips-mode-setup is disabled 98 | fips_disabled_code: 2 99 | -------------------------------------------------------------------------------- /roles/redpanda_broker/tests/tls_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import yaml 4 | from jinja2 import Environment, FileSystemLoader, Undefined 5 | 6 | class RecursiveUndefined(Undefined): 7 | def __getattr__(self, name): 8 | if name == 'undefined_name': 9 | return self._undefined_name 10 | return RecursiveUndefined(name=f'{self._undefined_name}.{name}') 11 | 12 | def __str__(self): 13 | return '{{ %s }}' % self._undefined_name 14 | 15 | class TestRedpandaTemplate(unittest.TestCase): 16 | def setUp(self): 17 | self.current_dir = os.path.dirname(os.path.abspath(__file__)) 18 | self.templates_dir = os.path.join(self.current_dir, '..', 'templates/configs') 19 | self.defaults_file = os.path.join(self.current_dir, '..', 'defaults', 'main.yml') 20 | 21 | # Load the defaults from the YAML file 22 | with open(self.defaults_file, 'r') as f: 23 | self.defaults = yaml.safe_load(f) 24 | 25 | # Create a Jinja2 environment with recursive undefined handling 26 | self.env = Environment( 27 | loader=FileSystemLoader(self.templates_dir), 28 | undefined=RecursiveUndefined 29 | ) 30 | 31 | # Add a custom filter for recursive rendering 32 | self.env.filters['recursive_render'] = self.recursive_render 33 | 34 | self.maxDiff = None 35 | self.hostvars = { 36 | '35.91.106.231': { 37 | 'ansible_host': '35.91.106.231', 38 | 'advertised_ip': '35.91.106.231', 39 | 'private_ip': '192.168.1.1' 40 | }, 41 | '35.88.129.205': { 42 | 'ansible_host': '35.88.129.205', 43 | 'advertised_ip': '35.88.129.205', 44 | 'private_ip': '192.168.1.2' 45 | }, 46 | '54.190.184.126': { 47 | 'ansible_host': '54.190.184.126', 48 | 'advertised_ip': '54.190.184.126', 49 | 'private_ip': '192.168.1.3' 50 | } 51 | } 52 | self.groups = { 53 | 'redpanda': ['35.91.106.231', '35.88.129.205', '54.190.184.126'] 54 | } 55 | 56 | def recursive_render(self, template_string): 57 | template = self.env.from_string(template_string) 58 | return template.render(**self.defaults) 59 | 60 | def test_redpanda_template_tls(self): 61 | template = self.env.get_template('tls.j2') 62 | 63 | # First pass: render the template 64 | first_pass = template.render( 65 | hostvars=self.hostvars, 66 | groups=self.groups, 67 | inventory_hostname='35.91.106.231', 68 | **self.defaults 69 | ) 70 | 71 | # Second pass: resolve any remaining Jinja2 expressions 72 | rendered_template = self.env.from_string(first_pass).render(**self.defaults) 73 | 74 | expected_rendered_values = { 75 | "node": { 76 | "redpanda": { 77 | "admin_api_tls": { 78 | "enabled": True, 79 | "require_client_auth": False, 80 | "key_file": "/etc/redpanda/certs/node.key", 81 | "cert_file": "/etc/redpanda/certs/node.crt", 82 | "truststore_file": "/etc/redpanda/certs/truststore.pem" 83 | }, 84 | "kafka_api_tls": { 85 | "enabled": True, 86 | "require_client_auth": False, 87 | "key_file": "/etc/redpanda/certs/node.key", 88 | "cert_file": "/etc/redpanda/certs/node.crt", 89 | "truststore_file": "/etc/redpanda/certs/truststore.pem" 90 | }, 91 | "rpc_server_tls": { 92 | "enabled": True, 93 | "require_client_auth": False, 94 | "key_file": "/etc/redpanda/certs/node.key", 95 | "cert_file": "/etc/redpanda/certs/node.crt", 96 | "truststore_file": "/etc/redpanda/certs/truststore.pem" 97 | } 98 | }, 99 | "rpk": { 100 | "admin_api": { 101 | "tls": { 102 | "truststore_file": "/etc/redpanda/certs/truststore.pem" 103 | } 104 | }, 105 | "kafka_api": { 106 | "tls": { 107 | "truststore_file": "/etc/redpanda/certs/truststore.pem" 108 | } 109 | } 110 | }, 111 | "pandaproxy": { 112 | "pandaproxy_api_tls": [ 113 | { 114 | "enabled": True, 115 | "require_client_auth": False, 116 | "key_file": "/etc/redpanda/certs/node.key", 117 | "cert_file": "/etc/redpanda/certs/node.crt", 118 | "truststore_file": "/etc/redpanda/certs/truststore.pem" 119 | } 120 | ] 121 | }, 122 | "schema_registry": { 123 | "schema_registry_api_tls": [ 124 | { 125 | "enabled": True, 126 | "require_client_auth": False, 127 | "key_file": "/etc/redpanda/certs/node.key", 128 | "cert_file": "/etc/redpanda/certs/node.crt", 129 | "truststore_file": "/etc/redpanda/certs/truststore.pem" 130 | } 131 | ] 132 | } 133 | } 134 | } 135 | 136 | rendered_dict = yaml.safe_load(rendered_template) 137 | self.assertEqual(rendered_dict, expected_rendered_values) 138 | 139 | print(rendered_template) 140 | 141 | if __name__ == '__main__': 142 | unittest.main() 143 | -------------------------------------------------------------------------------- /roles/redpanda_console/tests/pre_v3_defaults_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import yaml 4 | from jinja2 import Environment, FileSystemLoader 5 | import json 6 | 7 | 8 | class TestPreV3DefaultsTemplate(unittest.TestCase): 9 | def test_pre_v3_defaults_template(self): 10 | # Get the absolute path of the current file 11 | current_dir = os.path.dirname(os.path.abspath(__file__)) 12 | 13 | # Construct the path to the templates directory 14 | templates_dir = os.path.join(current_dir, '..', 'templates') 15 | 16 | # Construct the path to the defaults/main.yml file 17 | defaults_file = os.path.join(current_dir, '..', 'defaults', 'main.yml') 18 | 19 | # Load the defaults from the YAML file 20 | with open(defaults_file, 'r') as f: 21 | defaults = yaml.safe_load(f) 22 | 23 | # Create a Jinja2 environment and load the template from file 24 | env = Environment(loader=FileSystemLoader(templates_dir)) 25 | template = env.get_template('pre_v3_defaults.j2') 26 | 27 | # Define the hostvars and groups for rendering the template 28 | hostvars = { 29 | '35.91.106.231': { 30 | 'ansible_host': '35.91.106.231', 31 | }, 32 | '35.88.129.205': { 33 | 'ansible_host': '35.88.129.205', 34 | }, 35 | '54.190.184.126': { 36 | 'ansible_host': '54.190.184.126', 37 | } 38 | } 39 | advertised_ips = ["35.91.106.231", "35.88.129.205", "54.190.184.126"] 40 | groups = { 41 | 'redpanda': ['35.91.106.231', '35.88.129.205', '54.190.184.126'] 42 | } 43 | 44 | # Set up some test cases for enable_tls and connect presence 45 | scenarios = [ 46 | {'enable_tls': False, 'kafka_connect_present': False, 'name': 'without TLS, no connect'}, 47 | {'enable_tls': True, 'kafka_connect_present': False, 'name': 'with TLS, no connect'}, 48 | {'enable_tls': False, 'kafka_connect_present': True, 'name': 'without TLS, with connect'}, 49 | {'enable_tls': True, 'kafka_connect_present': True, 'name': 'with TLS, with connect'} 50 | ] 51 | 52 | for scenario in scenarios: 53 | # Merge scenario settings with defaults 54 | test_vars = {**defaults, **{'enable_tls': scenario['enable_tls']}} 55 | 56 | # Add kafka_connect_advertised_ips if connect is present 57 | if scenario['kafka_connect_present']: 58 | test_vars['kafka_connect_advertised_ips'] = advertised_ips 59 | groups['connect'] = groups['redpanda'] 60 | 61 | # Render the template 62 | rendered_template = template.render( 63 | hostvars=hostvars, 64 | advertised_ips=advertised_ips, 65 | groups=groups, 66 | **test_vars 67 | ) 68 | 69 | # Parse the rendered JSON to validate its structure 70 | try: 71 | parsed_json = json.loads(rendered_template) 72 | 73 | # Test that expected sections exist in the JSON 74 | self.assertIn("kafka", parsed_json) 75 | self.assertIn("brokers", parsed_json["kafka"]) 76 | self.assertIn("redpanda", parsed_json) 77 | self.assertIn("adminApi", parsed_json["redpanda"]) 78 | self.assertIn("schemaRegistry", parsed_json["kafka"]) 79 | self.assertIn("protobuf", parsed_json["kafka"]) 80 | 81 | # Test that TLS settings are present when enable_tls is True 82 | if scenario['enable_tls']: 83 | self.assertIn("tls", parsed_json["kafka"]) 84 | self.assertTrue(parsed_json["kafka"]["tls"]["enabled"]) 85 | self.assertIn("tls", parsed_json["redpanda"]["adminApi"]) 86 | self.assertTrue(parsed_json["redpanda"]["adminApi"]["tls"]["enabled"]) 87 | self.assertIn("server", parsed_json) 88 | self.assertIn("tls", parsed_json["server"]) 89 | self.assertTrue(parsed_json["server"]["tls"]["enabled"]) 90 | else: 91 | self.assertNotIn("tls", parsed_json.get("kafka", {})) 92 | 93 | # Test kafka brokers 94 | self.assertEqual(len(parsed_json["kafka"]["brokers"]), 3) 95 | for i, ip in enumerate(advertised_ips): 96 | expected_broker = f"{ip}:{defaults['redpanda_kafka_port']}" 97 | self.assertEqual(parsed_json["kafka"]["brokers"][i], expected_broker) 98 | 99 | # Test admin API URLs 100 | admin_urls = parsed_json["redpanda"]["adminApi"]["urls"] 101 | self.assertEqual(len(admin_urls), 3) 102 | protocol = "https" if scenario['enable_tls'] else "http" 103 | for i, ip in enumerate(advertised_ips): 104 | expected_url = f"{protocol}://{ip}:{defaults['redpanda_admin_api_port']}" 105 | self.assertEqual(admin_urls[i], expected_url) 106 | 107 | # Test schema registry URLs 108 | schema_urls = parsed_json["kafka"]["schemaRegistry"]["urls"] 109 | self.assertEqual(len(schema_urls), 3) 110 | for i, ip in enumerate(advertised_ips): 111 | expected_url = f"{protocol}://{ip}:{defaults['redpanda_schema_registry_port']}" 112 | self.assertEqual(schema_urls[i], expected_url) 113 | 114 | # Test connect settings if enabled 115 | if scenario['kafka_connect_present']: 116 | self.assertIn("connect", parsed_json) 117 | self.assertTrue(parsed_json["connect"]["enabled"]) 118 | self.assertEqual(len(parsed_json["connect"]["clusters"]), 1) 119 | self.assertEqual(parsed_json["connect"]["clusters"][0]["name"], "Connect") 120 | 121 | # Check connect URL 122 | connect_url = parsed_json["connect"]["clusters"][0]["url"] 123 | expected_url = f"{protocol}://{advertised_ips[0]}:{defaults['kafka_connect_port']}" 124 | self.assertEqual(connect_url, expected_url) 125 | 126 | # Check TLS settings in connect if applicable 127 | if scenario['enable_tls']: 128 | self.assertIn("tls", parsed_json["connect"]["clusters"][0]) 129 | self.assertTrue(parsed_json["connect"]["clusters"][0]["tls"]["enabled"]) 130 | else: 131 | self.assertNotIn("connect", parsed_json) 132 | 133 | print(f"Successfully validated scenario: {scenario['name']}") 134 | 135 | except json.JSONDecodeError as e: 136 | self.fail(f"Failed to parse rendered JSON template: {e}\n{rendered_template}") 137 | 138 | # Print the rendered template for debugging 139 | print(rendered_template) 140 | 141 | 142 | if __name__ == '__main__': 143 | unittest.main() 144 | -------------------------------------------------------------------------------- /roles/binary_bundler/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # defaults 2 | redpanda_base_url: "https://dl.redpanda.com" 3 | download_directory: "/tmp" 4 | 5 | # required config from user: 6 | redpanda_version: "" 7 | is_using_unstable: false 8 | basearch: "" 9 | os_distribution: "" 10 | os_distribution_major_version: "" 11 | rpm_or_deb: "" 12 | 13 | is_post_split: "{{ redpanda_version >= '24.2' or redpanda_version == 'latest' }}" 14 | 15 | # the rpms 16 | rpm_package_suffix: "{{ '' if redpanda_version=='latest' else ('=' if ansible_os_family == 'Debian' else '-') + redpanda_version }}.{{ basearch }}" 17 | rpm_base_url: "{{ redpanda_base_url }}/public/redpanda/rpm/{{ (os_distribution == 'RedHat') | ternary('el', os_distribution | lower) }}/{{ os_distribution_major_version }}" 18 | rpm_unstable_base_url: "{{ redpanda_base_url }}/E4xN1tVe3Xy60GTx/redpanda-unstable/rpm/{{ (os_distribution == 'RedHat') | ternary('el', os_distribution | lower) }}/{{ os_distribution_major_version }}" 19 | redpanda_package_name: "redpanda{{ rpm_package_suffix }}.rpm" 20 | redpanda_rpk_package_name: "redpanda-rpk{{ rpm_package_suffix }}.rpm" 21 | redpanda_tuner_package_name: "redpanda-tuner{{ rpm_package_suffix }}.rpm" 22 | 23 | rp_standard_rpm: "{{ rpm_base_url }}/{{ basearch }}/{{ redpanda_package_name }}" 24 | rp_standard_rpm_unstable: "{{ rpm_unstable_base_url }}/{{ basearch }}/{{ redpanda_package_name }}" 25 | rp_noarch_rpm: "{{ rpm_base_url }}/noarch/{{ redpanda_package_name }}" 26 | rp_noarch_rpm_unstable: "{{ rpm_unstable_base_url }}/noarch/{{ redpanda_package_name }}" 27 | rp_source_rpm: "{{ rpm_base_url }}/SRPMS/{{ redpanda_package_name }}" 28 | rp_source_rpm_unstable: "{{ rpm_unstable_base_url }}/SRPMS/{{ redpanda_package_name }}" 29 | 30 | rpk_standard_rpm: "{{ rpm_base_url }}/{{ basearch }}/{{ redpanda_rpk_package_name }}" 31 | rpk_standard_rpm_unstable: "{{ rpm_unstable_base_url }}/{{ basearch }}/{{ redpanda_rpk_package_name }}" 32 | rpk_noarch_rpm: "{{ rpm_base_url }}/noarch/{{ redpanda_rpk_package_name }}" 33 | rpk_noarch_rpm_unstable: "{{ rpm_unstable_base_url }}/noarch/{{ redpanda_rpk_package_name }}" 34 | rpk_source_rpm: "{{ rpm_base_url }}/SRPMS/{{ redpanda_rpk_package_name }}" 35 | rpk_source_rpm_unstable: "{{ rpm_unstable_base_url }}/SRPMS/{{ redpanda_rpk_package_name }}" 36 | 37 | tuner_standard_rpm: "{{ rpm_base_url }}/{{ basearch }}/{{ redpanda_tuner_package_name }}" 38 | tuner_standard_rpm_unstable: "{{ rpm_unstable_base_url }}/{{ basearch }}/{{ redpanda_tuner_package_name }}" 39 | tuner_noarch_rpm: "{{ rpm_base_url }}/noarch/{{ redpanda_tuner_package_name }}" 40 | tuner_noarch_rpm_unstable: "{{ rpm_unstable_base_url }}/noarch/{{ redpanda_tuner_package_name }}" 41 | tuner_source_rpm: "{{ rpm_base_url }}/SRPMS/{{ redpanda_tuner_package_name }}" 42 | tuner_source_rpm_unstable: "{{ rpm_unstable_base_url }}/SRPMS/{{ redpanda_tuner_package_name }}" 43 | 44 | pre_split_standard_rpms: 45 | redpanda: "{{ rp_standard_rpm }}" 46 | 47 | pre_split_standard_rpms_unstable: 48 | redpanda: "{{ rp_standard_rpm_unstable }}" 49 | 50 | pre_split_noarch_rpms: 51 | redpanda: "{{ rp_noarch_rpm }}" 52 | 53 | pre_split_noarch_rpms_unstable: 54 | redpanda: "{{ rp_noarch_rpm_unstable }}" 55 | 56 | pre_split_source_rpms: 57 | redpanda: "{{ rp_source_rpm }}" 58 | 59 | pre_split_source_rpms_unstable: 60 | redpanda: "{{ rp_source_rpm_unstable }}" 61 | 62 | post_split_standard_rpms: 63 | redpanda: "{{ rp_standard_rpm }}" 64 | redpanda-rpk: "{{ rpk_standard_rpm }}" 65 | redpanda-tuner: "{{ tuner_standard_rpm }}" 66 | 67 | post_split_standard_rpms_unstable: 68 | redpanda: "{{ rp_standard_rpm_unstable }}" 69 | redpanda-rpk: "{{ rpk_standard_rpm_unstable }}" 70 | redpanda-tuner: "{{ tuner_standard_rpm_unstable }}" 71 | 72 | post_split_noarch_rpms: 73 | redpanda: "{{ rp_noarch_rpm }}" 74 | redpanda-rpk: "{{ rpk_noarch_rpm }}" 75 | redpanda-tuner: "{{ tuner_noarch_rpm }}" 76 | 77 | post_split_noarch_rpms_unstable: 78 | redpanda: "{{ rp_noarch_rpm_unstable }}" 79 | redpanda-rpk: "{{ rpk_noarch_rpm_unstable }}" 80 | redpanda-tuner: "{{ tuner_noarch_rpm_unstable }}" 81 | 82 | post_split_source_rpms: 83 | redpanda: "{{ rp_source_rpm }}" 84 | redpanda-rpk: "{{ rpk_source_rpm }}" 85 | redpanda-tuner: "{{ tuner_source_rpm }}" 86 | 87 | post_split_source_rpms_unstable: 88 | redpanda: "{{ rp_source_rpm_unstable }}" 89 | redpanda-rpk: "{{ rpk_source_rpm_unstable }}" 90 | redpanda-tuner: "{{ tuner_source_rpm_unstable }}" 91 | 92 | rpm_standard_download_urls: "{{ post_split_standard_rpms if is_post_split else pre_split_standard_rpms }}" 93 | rpm_noarch_download_urls: "{{ post_split_noarch_rpms if is_post_split else pre_split_noarch_rpms }}" 94 | rpm_source_download_urls: "{{ post_split_source_rpms if is_post_split else pre_split_source_rpms }}" 95 | rpm_standard_unstable_download_urls: "{{ post_split_standard_rpms_unstable if is_post_split else pre_split_standard_rpms_unstable }}" 96 | rpm_noarch_unstable_download_urls: "{{ post_split_noarch_rpms_unstable if is_post_split else pre_split_noarch_rpms_unstable }}" 97 | rpm_source_unstable_download_urls: "{{ post_split_source_rpms_unstable if is_post_split else pre_split_source_rpms_unstable }}" 98 | 99 | # the debs 100 | deb_download_url: "https://dl.redpanda.com/public/redpanda/deb/{{ os_distribution }}/pool/any-version/main/r/re/redpanda_{{ redpanda_version}}/redpanda_{{ redpanda_version }}_{{ basearch }}.deb" 101 | deb_unstable_download_url: "https://dl.redpanda.com/E4xN1tVe3Xy60GTx/redpanda-unstable/deb/{{ os_distribution }}/pool/any-version/main/r/re/redpanda_{{ redpanda_version}}/redpanda_{{ redpanda_version }}_{{ basearch }}.deb" 102 | 103 | pre_split_deb_download_urls: 104 | redpanda: "{{ deb_download_url }}" 105 | 106 | pre_split_deb_unstable_download_urls: 107 | redpanda: "{{ deb_unstable_download_url }}" 108 | 109 | post_split_deb_download_urls: 110 | redpanda-rpk: "https://dl.redpanda.com/public/redpanda/deb/{{ os_distribution }}/pool/any-version/main/r/re/redpanda-rpk_{{ redpanda_version}}/redpanda-rpk_{{ redpanda_version }}_{{ basearch }}.deb" 111 | redpanda-tuner: "https://dl.redpanda.com/public/redpanda/deb/{{ os_distribution }}/pool/any-version/main/r/re/redpanda-tuner_{{ redpanda_version}}/redpanda-tuner_{{ redpanda_version }}_{{ basearch }}.deb" 112 | redpanda: "https://dl.redpanda.com/public/redpanda/deb/{{ os_distribution }}/pool/any-version/main/r/re/redpanda_{{ redpanda_version}}/redpanda_{{ redpanda_version }}_{{ basearch }}.deb" 113 | 114 | post_split_deb_unstable_download_urls: 115 | redpanda-rpk: "https://dl.redpanda.com/E4xN1tVe3Xy60GTx/redpanda-unstable/deb/{{ os_distribution }}/pool/any-version/main/r/re/redpanda-rpk_{{ redpanda_version}}/redpanda-rpk_{{ redpanda_version }}_{{ basearch }}.deb" 116 | redpanda-tuner: "https://dl.redpanda.com/E4xN1tVe3Xy60GTx/redpanda-unstable/deb/{{ os_distribution }}/pool/any-version/main/r/re/redpanda-tuner_{{ redpanda_version}}/redpanda-tuner_{{ redpanda_version }}_{{ basearch }}.deb" 117 | redpanda: "https://dl.redpanda.com/E4xN1tVe3Xy60GTx/redpanda-unstable/deb/{{ os_distribution }}/pool/any-version/main/r/re/redpanda_{{ redpanda_version}}/redpanda_{{ redpanda_version }}_{{ basearch }}.deb" 118 | 119 | deb_download_urls: "{{ post_split_deb_download_urls if is_post_split else pre_split_deb_download_urls }}" 120 | deb_unstable_download_urls: "{{ post_split_deb_unstable_download_urls if is_post_split else post_split_deb_unstable_download_urls }}" 121 | --------------------------------------------------------------------------------