├── .gitignore ├── .pyup.yml ├── .travis.yml ├── .yamllint ├── CHANGELOG.md ├── LICENSE ├── README.md ├── defaults └── main.yml ├── files └── .keep ├── filter_plugins └── __init__.py ├── handlers └── main.yml ├── meta └── main.yml ├── molecule └── default │ ├── Dockerfile.j2 │ ├── INSTALL.rst │ ├── molecule.yml │ ├── playbook.yml │ └── tests │ └── test_installation.py ├── requirements.txt ├── tasks ├── main.yml ├── manage_airflow_variables.yml ├── manage_configuration.yml ├── manage_connections.yml ├── manage_installation.yml ├── manage_services.yml └── manage_variables.yml ├── templates ├── .keep ├── airflow.cfg.j2 ├── airflow.env.j2 ├── airflow.profile.j2 ├── logrotate.j2 └── systemd.service.j2 ├── tests ├── __init__.py ├── playbook.yml └── test_filter_plugins.py ├── tox.ini └── vars ├── main.yml ├── os_distribution ├── .keep ├── debian │ └── stretch.yml └── ubuntu │ ├── bionic.yml │ └── xenial.yml └── os_family ├── .keep └── debian.yml /.gitignore: -------------------------------------------------------------------------------- 1 | # Ansible 2 | *.retry 3 | roles/ 4 | tests/roles/ 5 | 6 | # Swap files 7 | *.swo 8 | *.swp 9 | 10 | # Python bytecode 11 | *.pyc 12 | 13 | # Pytest 14 | .cache 15 | 16 | # Tox 17 | .tox 18 | 19 | # Vagrant 20 | .vagrant 21 | *.log 22 | 23 | # Molecule 24 | .molecule 25 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Pyup configuration 4 | 5 | branch: 'develop' 6 | 7 | requirements: 8 | - 'requirements.txt' 9 | - 'tox.ini' 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Travis configuration 4 | 5 | dist: 'trusty' 6 | sudo: 'required' 7 | 8 | services: 9 | - 'docker' 10 | 11 | language: 'python' 12 | 13 | env: 14 | matrix: 15 | - TOXENV='py27-ansible24' 16 | - TOXENV='py27-ansible25' 17 | 18 | install: 19 | - 'pip install tox-travis' 20 | 21 | script: 22 | - 'travis_wait 50 tox -e "${TOXENV}"' 23 | 24 | notifications: 25 | webhooks: 'https://galaxy.ansible.com/api/v1/notifications/' 26 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | extends: default 2 | 3 | rules: 4 | braces: 5 | max-spaces-inside: 1 6 | level: error 7 | brackets: 8 | max-spaces-inside: 1 9 | level: error 10 | line-length: disable 11 | truthy: disable 12 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Describe changes between releases 2 | 3 | ## 0.2.0 4 | 5 | This is one of the last 0.x release, role structure is validated and mature 6 | 7 | * Add connections management 8 | * Update Molecule to 2.x for role testing 9 | * Update Airlow default version to 1.8.2 10 | * System dependencies are managed by distribution/family dedicated files 11 | * Manage also Debian Jessie and Ubuntu Xenial deployment 12 | * Replace `airflow_default_system_dependencies` by `_airflow_system_dependencies` 13 | * `airflow_system_dependencies` default value is now `_airflow_system_dependencies` 14 | You can use combine filter to add your packages if needed 15 | * Update core section "home" key to "airflow_home" 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 infOpen 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # airflow 2 | 3 | [![Build Status](https://img.shields.io/travis/infOpen/ansible-role-airflow/master.svg?label=travis_master)](https://travis-ci.org/infOpen/ansible-role-airflow) 4 | [![Build Status](https://img.shields.io/travis/infOpen/ansible-role-airflow/develop.svg?label=travis_develop)](https://travis-ci.org/infOpen/ansible-role-airflow) 5 | [![Updates](https://pyup.io/repos/github/infOpen/ansible-role-airflow/shield.svg)](https://pyup.io/repos/github/infOpen/ansible-role-airflow/) 6 | [![Python 3](https://pyup.io/repos/github/infOpen/ansible-role-airflow/python-3-shield.svg)](https://pyup.io/repos/github/infOpen/ansible-role-airflow/) 7 | [![Ansible Role](https://img.shields.io/ansible/role/10445.svg)](https://galaxy.ansible.com/infOpen/airflow/) 8 | 9 | Ansible role to manage Airflow installation and configuration 10 | 11 | First role usage is to manage a single master instance, so I've not manage worker side. If you want, free to do PR to add these features. 12 | 13 | ## Breaking changes 14 | 15 | ### 0.4.0 16 | 17 | * Remove Ubuntu Trusty management 18 | * Remove Ansible 2.2 and 2.3 management 19 | * Default Airflow version is now 1.9.0 20 | 21 | ## Requirements 22 | 23 | This role requires Ansible 2.4 or higher, 24 | and platform requirements are listed in the metadata file. 25 | 26 | ## Testing 27 | 28 | This role use [Molecule](https://github.com/metacloud/molecule/) to run tests. 29 | 30 | Local and Travis tests run tests on Docker by default. 31 | See molecule documentation to use other backend. 32 | 33 | Currently, tests are done on: 34 | - Debian Stretch 35 | - Ubuntu Xenial 36 | - Ubuntu Bionic 37 | 38 | and use: 39 | - Ansible 2.4.x 40 | - Ansible 2.5.x 41 | 42 | ### Running tests 43 | 44 | #### Using Docker driver 45 | 46 | ``` 47 | $ tox 48 | ``` 49 | 50 | ## Role Variables 51 | 52 | > **Warning** 53 | > No Fernet key defined on configuration, so set your own before store passwords ! 54 | 55 | ### Default role variables 56 | 57 | ``` yaml 58 | --- 59 | 60 | # Defaults vars file for airflow role 61 | 62 | # Airflow installation tasks 63 | airflow_system_dependencies: "{{ _airflow_system_dependencies }}" 64 | 65 | # Set changed when due to idempotency problem with pip module 66 | # Always changed with airflow and airflow[crypto] 67 | airflow_pip_changed_when: False 68 | 69 | # Installation vars 70 | airflow_user_name: 'airflow' 71 | airflow_user_group: "{{ airflow_user_name }}" 72 | airflow_user_shell: '/bin/false' 73 | airflow_user_home_path: '/var/lib/airflow' 74 | airflow_user_home_mode: '0700' 75 | 76 | airflow_log_path: '/var/log/airflow' 77 | airflow_log_owner: "{{ airflow_user_name }}" 78 | airflow_log_group: "{{ airflow_user_group }}" 79 | airflow_log_mode: '0700' 80 | 81 | airflow_pid_path: '/var/run/airflow' 82 | airflow_pid_owner: "{{ airflow_user_name }}" 83 | airflow_pid_group: "{{ airflow_user_group }}" 84 | airflow_pid_mode: '0700' 85 | 86 | airflow_virtualenv: "{{ airflow_user_home_path }}/venv" 87 | airflow_python_version: "{{ _airflow_python_version }}" 88 | airflow_packages: 89 | - name: 'pip' 90 | version: '10.0.1' 91 | - name: 'setuptools' 92 | version: '39.1.0' 93 | - name: 'GitPython' 94 | version: '2.1.9' 95 | - name: 'Cython' 96 | version: '0.28.2' 97 | - name: 'apache-airflow' 98 | version: '1.9.0' 99 | airflow_extra_packages: 100 | - name: 'apache-airflow[crypto]' 101 | 102 | 103 | # SERVICES MANAGEMENT 104 | # ----------------------------------------------------------------------------- 105 | 106 | # Airflow systemd services specific settings 107 | is_systemd_managed_system: "{{ _is_systemd_managed_system | default(False) }}" 108 | airflow_services_systemd: 109 | - dest: '/etc/systemd/system/airflow-webserver.service' 110 | handler: 'Restart airflow-webserver' 111 | options: 112 | Install: 113 | WantedBy: 'multi-user.target' 114 | Service: 115 | Environment: "PATH={{ airflow_virtualenv ~ '/bin' }}" 116 | EnvironmentFile: "{{ airflow_paths.files.environment.path }}" 117 | ExecStart: "{{ airflow_virtualenv ~ '/bin' }}/airflow webserver" 118 | Group: "{{ airflow_user_group }}" 119 | PrivateTmp: 'true' 120 | Restart: 'on-failure' 121 | RestartSec: '5s' 122 | Type: 'simple' 123 | User: "{{ airflow_user_name }}" 124 | Unit: 125 | After: 'network.target postgresql.service mysql.service redis.service rabbitmq-server.service' 126 | Description: 'Airflow webserver daemon' 127 | Wants: 'postgresql.service mysql.service redis.service rabbitmq-server.service' 128 | - dest: '/etc/systemd/system/airflow-scheduler.service' 129 | handler: 'Restart airflow-scheduler' 130 | options: 131 | Install: 132 | WantedBy: 'multi-user.target' 133 | Service: 134 | Environment: "PATH={{ airflow_virtualenv ~ '/bin' }}" 135 | EnvironmentFile: "{{ airflow_paths.files.environment.path }}" 136 | ExecStart: "{{ airflow_virtualenv ~ '/bin' }}/airflow scheduler" 137 | Group: "{{ airflow_user_group }}" 138 | PrivateTmp: 'true' 139 | Restart: 'always' 140 | RestartSec: '5s' 141 | Type: 'simple' 142 | User: "{{ airflow_user_name }}" 143 | Unit: 144 | After: 'network.target postgresql.service mysql.service redis.service rabbitmq-server.service' 145 | Description: 'Airflow scheduler daemon' 146 | Wants: 'postgresql.service mysql.service redis.service rabbitmq-server.service' 147 | 148 | airflow_services_states: 149 | - name: 'airflow-webserver' 150 | enabled: True 151 | state: 'started' 152 | - name: 'airflow-scheduler' 153 | enabled: True 154 | state: 'started' 155 | 156 | # Environment variables file 157 | airflow_paths: 158 | files: 159 | environment: 160 | path: '/etc/default/airflow' 161 | 162 | # Databases variables 163 | airflow_manage_database: True 164 | airflow_database_engine: 'mysql' 165 | 166 | # Set do_init to false if database already initialized 167 | airflow_do_init_db: True 168 | airflow_do_upgrade_db: True 169 | 170 | # Default configuration 171 | airflow_defaults_config: 172 | core: 173 | airflow_home: "{{ airflow_user_home_path ~ '/airflow' }}" 174 | dags_folder: "{{ airflow_user_home_path ~ '/airflow/dags' }}" 175 | base_log_folder: "{{ airflow_log_path }}" 176 | remote_base_log_folder: '' 177 | remote_log_conn_id: '' 178 | encrypt_s3_logs: False 179 | executor: 'SequentialExecutor' 180 | sql_alchemy_conn: 'sqlite:////var/lib/airflow/airflow/airflow.db' 181 | sql_alchemy_pool_size: 5 182 | sql_alchemy_pool_recycle: 3600 183 | parallelism: 32 184 | dag_concurrency: 16 185 | dags_are_paused_at_creation: True 186 | non_pooled_task_slot_count: 128 187 | max_active_runs_per_dag: 16 188 | load_examples: False 189 | plugins_folder: "{{ airflow_user_home_path ~ '/airflow/plugins' }}" 190 | fernet_key: 'cryptography_not_found_storing_passwords_in_plain_text' 191 | donot_pickle: False 192 | dagbag_import_timeout: 30 193 | 194 | operators: 195 | default_owner: 'Airflow' 196 | 197 | webserver: 198 | auth_backend: '' 199 | authenticate: False 200 | base_url: 'http://localhost:8080' 201 | debug: False 202 | expose_config: True 203 | filter_by_owner: False 204 | hostname: "{{ ansible_default_ipv4.address }}" 205 | log_file: "{{ airflow_log_path }}/airflow-webserver.log" 206 | pid: "{{ airflow_pid_path }}/airflow-webserver.pid" 207 | secret_key: 'temporary_key' 208 | web_server_host: '0.0.0.0' 209 | web_server_port: '8080' 210 | web_server_worker_timeout: 120 211 | workers: 1 212 | worker_class: 'sync' 213 | worker_timeout: 30 214 | email: 215 | email_backend: 'airflow.utils.email.send_email_smtp' 216 | 217 | smtp: 218 | smtp_host: 'localhost' 219 | smtp_starttls: True 220 | smtp_ssl: False 221 | smtp_user: 'airflow' 222 | smtp_port: 25 223 | smtp_password: 'airflow' 224 | smtp_mail_from: 'airflow@airflow.com' 225 | 226 | celery: 227 | celery_app_name: 'airflow.executors.celery_executor' 228 | celeryd_concurrency: 16 229 | worker_log_server_port: 8793 230 | broker_url: 'sqla+mysql://airflow:airflow@localhost:3306/airflow' 231 | celery_result_backend: 'db+mysql://airflow:airflow@localhost:3306/airflow' 232 | flower_port: 5555 233 | default_queue: 'default' 234 | 235 | scheduler: 236 | job_heartbeat_sec: 5 237 | log_file: "{{ airflow_log_path }}/airflow-scheduler.log" 238 | num_runs: 0 239 | pid: "{{ airflow_pid_path }}/airflow-scheduler.pid" 240 | scheduler_heartbeat_sec: 5 241 | statsd_on: False 242 | statsd_host: 'localhost' 243 | statsd_port: 8125 244 | statsd_prefix: 'airflow' 245 | max_threads: 2 246 | 247 | mesos: 248 | master: 'localhost:5050' 249 | framework_name: 'Airflow' 250 | task_cpu: 1 251 | task_memory: 256 252 | checkpoint: False 253 | failover_timeout: 604800 254 | authenticate: False 255 | default_principal: 'admin' 256 | default_secret: 'admin' 257 | 258 | ldap: 259 | uri: 'ldaps://:' 260 | user_filter: 'objectClass=*' 261 | user_name_attr: 'uid' 262 | superuser_filter: '' 263 | data_profiler_filter: '' 264 | bind_user: '' 265 | bind_password: '' 266 | basedn: 'dc=example,dc=com' 267 | cacert: '' 268 | search_scope: 'LEVEL' 269 | 270 | airflow_user_config: {} 271 | airflow_config: "{{ airflow_defaults_config | combine(airflow_user_config, recursive=True) }}" 272 | 273 | # Connections management 274 | airflow_drop_existing_connections_before_add: True 275 | airflow_connections: [] 276 | 277 | # Variables management 278 | airflow_drop_existing_variables_before_add: True 279 | airflow_variables: [] 280 | 281 | # Logrotate configuration 282 | airflow_logrotate_config: 283 | - filename: '/etc/logrotate.d/airflow' 284 | log_pattern: 285 | - "{{ airflow_log_path }}/*.log" 286 | options: 287 | - 'rotate 12' 288 | - 'weekly' 289 | - 'compress' 290 | - 'delaycompress' 291 | - "create 640 {{ airflow_log_owner }} {{ airflow_log_group }}" 292 | - 'postrotate' 293 | - 'endscript' 294 | ``` 295 | 296 | ## Dependencies 297 | 298 | None 299 | 300 | ## Example Playbook 301 | 302 | ``` yaml 303 | - hosts: servers 304 | roles: 305 | - { role: infOpen.airflow } 306 | ``` 307 | 308 | ## License 309 | 310 | MIT 311 | 312 | ## Author Information 313 | 314 | Alexandre Chaussier (for Infopen company) 315 | - http://www.infopen.pro 316 | - a.chaussier [at] infopen.pro 317 | -------------------------------------------------------------------------------- /defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Defaults vars file for airflow role 4 | 5 | # Airflow installation tasks 6 | airflow_system_dependencies: "{{ _airflow_system_dependencies }}" 7 | 8 | # Set changed when due to idempotency problem with pip module 9 | # Always changed with airflow and airflow[crypto] 10 | airflow_pip_changed_when: False 11 | 12 | # Ansible's pip module doesn't currently support complex version strings 13 | # https://github.com/ansible/ansible/issues/19321 14 | # And Airflow may currently require custom version strings 15 | # https://stackoverflow.com/a/48075827 16 | airflow_pip_custom_version_install: False 17 | 18 | # Installation vars 19 | airflow_user_name: 'airflow' 20 | airflow_user_group: "{{ airflow_user_name }}" 21 | airflow_user_shell: '/bin/false' 22 | airflow_user_home_path: '/var/lib/airflow' 23 | airflow_user_home_mode: '0700' 24 | 25 | airflow_log_path: '/var/log/airflow' 26 | airflow_log_owner: "{{ airflow_user_name }}" 27 | airflow_log_group: "{{ airflow_user_group }}" 28 | airflow_log_mode: '0700' 29 | 30 | airflow_pid_path: '/var/run/airflow' 31 | airflow_pid_owner: "{{ airflow_user_name }}" 32 | airflow_pid_group: "{{ airflow_user_group }}" 33 | airflow_pid_mode: '0700' 34 | 35 | airflow_virtualenv: "{{ airflow_user_home_path }}/venv" 36 | airflow_virtualenv_mode: "0755" 37 | airflow_python_version: "{{ _airflow_python_version }}" 38 | airflow_packages: 39 | - name: 'pip' 40 | version: '10.0.1' 41 | - name: 'setuptools' 42 | version: '39.1.0' 43 | - name: 'GitPython' 44 | version: '2.1.9' 45 | - name: 'Cython' 46 | version: '0.28.2' 47 | - name: 'apache-airflow' 48 | version: '1.9.0' 49 | airflow_extra_packages: 50 | - name: 'apache-airflow[crypto]' 51 | 52 | # option to set environment for all users, 53 | # by providing a file in /etc/profile.d/ 54 | airflow_provide_user_env: False 55 | 56 | 57 | # SERVICES MANAGEMENT 58 | # ----------------------------------------------------------------------------- 59 | 60 | # Airflow systemd services specific settings 61 | is_systemd_managed_system: "{{ _is_systemd_managed_system | default(False) }}" 62 | airflow_services_systemd: 63 | - dest: '/etc/systemd/system/airflow-webserver.service' 64 | handler: 'Restart airflow-webserver' 65 | options: 66 | Install: 67 | WantedBy: 'multi-user.target' 68 | Service: 69 | Environment: "PATH={{ airflow_virtualenv ~ '/bin' }}" 70 | EnvironmentFile: "{{ airflow_paths.files.environment.path }}" 71 | ExecStart: "{{ airflow_virtualenv ~ '/bin' }}/airflow webserver" 72 | Group: "{{ airflow_user_group }}" 73 | PrivateTmp: 'true' 74 | Restart: 'on-failure' 75 | RestartSec: '5s' 76 | Type: 'simple' 77 | User: "{{ airflow_user_name }}" 78 | Unit: 79 | After: 'network.target postgresql.service mysql.service redis.service rabbitmq-server.service' 80 | Description: 'Airflow webserver daemon' 81 | Wants: 'postgresql.service mysql.service redis.service rabbitmq-server.service' 82 | - dest: '/etc/systemd/system/airflow-scheduler.service' 83 | handler: 'Restart airflow-scheduler' 84 | options: 85 | Install: 86 | WantedBy: 'multi-user.target' 87 | Service: 88 | Environment: "PATH={{ airflow_virtualenv ~ '/bin' }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" 89 | EnvironmentFile: "{{ airflow_paths.files.environment.path }}" 90 | ExecStart: "{{ airflow_virtualenv ~ '/bin' }}/airflow scheduler" 91 | Group: "{{ airflow_user_group }}" 92 | PrivateTmp: 'true' 93 | Restart: 'always' 94 | RestartSec: '5s' 95 | Type: 'simple' 96 | User: "{{ airflow_user_name }}" 97 | Unit: 98 | After: 'network.target postgresql.service mysql.service redis.service rabbitmq-server.service' 99 | Description: 'Airflow scheduler daemon' 100 | Wants: 'postgresql.service mysql.service redis.service rabbitmq-server.service' 101 | 102 | airflow_services_states: 103 | - name: 'airflow-webserver' 104 | enabled: True 105 | state: 'started' 106 | - name: 'airflow-scheduler' 107 | enabled: True 108 | state: 'started' 109 | 110 | # Environment variables file 111 | airflow_paths: 112 | files: 113 | environment: 114 | path: '/etc/default/airflow' 115 | 116 | # Databases variables 117 | airflow_manage_database: True 118 | airflow_database_engine: 'mysql' 119 | airflow_database_sqlite_file_path: "{{ airflow_user_home_path ~ '/airflow/airflow.db' }}" 120 | 121 | # Set do_init to false if database already initialized 122 | airflow_do_init_db: True 123 | airflow_do_upgrade_db: True 124 | 125 | # Default configuration 126 | airflow_defaults_config: 127 | core: 128 | airflow_home: "{{ airflow_user_home_path ~ '/airflow' }}" 129 | airflow_home_mode: "0700" 130 | dags_folder: "{{ airflow_user_home_path ~ '/airflow/dags' }}" 131 | dags_folder_mode: "0755" 132 | base_log_folder: "{{ airflow_log_path }}" 133 | remote_base_log_folder: '' 134 | remote_log_conn_id: '' 135 | encrypt_s3_logs: False 136 | executor: 'SequentialExecutor' 137 | sql_alchemy_conn: "sqlite:///{{ airflow_database_sqlite_file_path }}" 138 | sql_alchemy_pool_size: 5 139 | sql_alchemy_pool_recycle: 3600 140 | parallelism: 32 141 | dag_concurrency: 16 142 | dags_are_paused_at_creation: True 143 | non_pooled_task_slot_count: 128 144 | max_active_runs_per_dag: 16 145 | load_examples: False 146 | plugins_folder: "{{ airflow_user_home_path ~ '/airflow/plugins' }}" 147 | fernet_key: 'cryptography_not_found_storing_passwords_in_plain_text' 148 | donot_pickle: False 149 | dagbag_import_timeout: 30 150 | 151 | operators: 152 | default_owner: 'Airflow' 153 | 154 | webserver: 155 | auth_backend: '' 156 | authenticate: False 157 | base_url: 'http://localhost:8080' 158 | debug: False 159 | expose_config: True 160 | filter_by_owner: False 161 | hostname: "{{ ansible_default_ipv4.address }}" 162 | log_file: "{{ airflow_log_path }}/airflow-webserver.log" 163 | pid: "{{ airflow_pid_path }}/airflow-webserver.pid" 164 | secret_key: 'temporary_key' 165 | web_server_host: '0.0.0.0' 166 | web_server_port: '8080' 167 | web_server_worker_timeout: 120 168 | workers: 1 169 | worker_class: 'sync' 170 | worker_timeout: 30 171 | email: 172 | email_backend: 'airflow.utils.email.send_email_smtp' 173 | 174 | smtp: 175 | smtp_host: 'localhost' 176 | smtp_starttls: True 177 | smtp_ssl: False 178 | smtp_user: 'airflow' 179 | smtp_port: 25 180 | smtp_password: 'airflow' 181 | smtp_mail_from: 'airflow@airflow.com' 182 | 183 | celery: 184 | celery_app_name: 'airflow.executors.celery_executor' 185 | celeryd_concurrency: 16 186 | worker_log_server_port: 8793 187 | broker_url: 'sqla+mysql://airflow:airflow@localhost:3306/airflow' 188 | celery_result_backend: 'db+mysql://airflow:airflow@localhost:3306/airflow' 189 | flower_port: 5555 190 | default_queue: 'default' 191 | 192 | scheduler: 193 | job_heartbeat_sec: 5 194 | log_file: "{{ airflow_log_path }}/airflow-scheduler.log" 195 | num_runs: 0 196 | pid: "{{ airflow_pid_path }}/airflow-scheduler.pid" 197 | scheduler_heartbeat_sec: 5 198 | statsd_on: False 199 | statsd_host: 'localhost' 200 | statsd_port: 8125 201 | statsd_prefix: 'airflow' 202 | max_threads: 2 203 | 204 | mesos: 205 | master: 'localhost:5050' 206 | framework_name: 'Airflow' 207 | task_cpu: 1 208 | task_memory: 256 209 | checkpoint: False 210 | failover_timeout: 604800 211 | authenticate: False 212 | default_principal: 'admin' 213 | default_secret: 'admin' 214 | 215 | ldap: 216 | uri: 'ldaps://:' 217 | user_filter: 'objectClass=*' 218 | user_name_attr: 'uid' 219 | superuser_filter: '' 220 | data_profiler_filter: '' 221 | bind_user: '' 222 | bind_password: '' 223 | basedn: 'dc=example,dc=com' 224 | cacert: '' 225 | search_scope: 'LEVEL' 226 | 227 | airflow_user_config: {} 228 | airflow_config: "{{ airflow_defaults_config | combine(airflow_user_config, recursive=True) }}" 229 | 230 | # Connections management 231 | airflow_drop_existing_connections_before_add: True 232 | airflow_connections: [] 233 | 234 | # Variables management 235 | airflow_drop_existing_variables_before_add: True 236 | airflow_variables: [] 237 | 238 | # Logrotate configuration 239 | airflow_logrotate_config: 240 | - filename: '/etc/logrotate.d/airflow' 241 | log_pattern: 242 | - "{{ airflow_log_path }}/*.log" 243 | options: 244 | - 'rotate 12' 245 | - 'weekly' 246 | - 'compress' 247 | - 'delaycompress' 248 | - "create 640 {{ airflow_log_owner }} {{ airflow_log_group }}" 249 | - 'postrotate' 250 | - 'endscript' 251 | -------------------------------------------------------------------------------- /files/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infOpen/ansible-role-airflow/23c6cf0f465fe34df6b0b12e60d344abbfdf4260/files/.keep -------------------------------------------------------------------------------- /filter_plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infOpen/ansible-role-airflow/23c6cf0f465fe34df6b0b12e60d344abbfdf4260/filter_plugins/__init__.py -------------------------------------------------------------------------------- /handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Handlers main file for airflow role 4 | 5 | - name: 'Restart airflow' 6 | become: True 7 | service: 8 | name: "{{ item.name }}" 9 | state: 'restarted' 10 | with_items: "{{ airflow_services_states }}" 11 | when: "item.state == 'started'" 12 | tags: 13 | - 'role::airflow' 14 | - 'role::airflow::config' 15 | - 'role::airflow::install' 16 | 17 | 18 | - name: 'Restart airflow-webserver' 19 | become: True 20 | service: 21 | name: "{{ item.name }}" 22 | state: 'restarted' 23 | with_items: "{{ airflow_services_states }}" 24 | when: 25 | - "item.name == 'airflow-webserver'" 26 | - "item.state == 'started'" 27 | tags: 28 | - 'role::airflow' 29 | - 'role::airflow::config' 30 | - 'role::airflow::install' 31 | 32 | 33 | - name: 'Restart airflow-scheduler' 34 | become: True 35 | service: 36 | name: "{{ item.name }}" 37 | state: 'restarted' 38 | with_items: "{{ airflow_services_states }}" 39 | when: 40 | - "item.name == 'airflow-scheduler'" 41 | - "item.state == 'started'" 42 | tags: 43 | - 'role::airflow' 44 | - 'role::airflow::config' 45 | - 'role::airflow::install' 46 | 47 | 48 | - name: 'Restart airflow-kerberos' 49 | become: True 50 | service: 51 | name: "{{ item.name }}" 52 | state: 'restarted' 53 | with_items: "{{ airflow_services_states }}" 54 | when: 55 | - "item.name == 'airflow-kerberos'" 56 | - "item.state == 'started'" 57 | tags: 58 | - 'role::airflow' 59 | - 'role::airflow::config' 60 | - 'role::airflow::install' 61 | -------------------------------------------------------------------------------- /meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | galaxy_info: 4 | author: 'Alexandre Chaussier' 5 | description: 'Ansible role to install and configure airflow' 6 | company: 'Infopen (http://www.infopen.pro)' 7 | license: 'MIT' 8 | min_ansible_version: '2.4' 9 | github_branch: 'master' 10 | platforms: 11 | - name: 'Debian' 12 | versions: 13 | - 'stretch' 14 | - name: 'Ubuntu' 15 | versions: 16 | - 'xenial' 17 | - 'bionic' 18 | galaxy_tags: 19 | - 'system' 20 | dependencies: [] 21 | -------------------------------------------------------------------------------- /molecule/default/Dockerfile.j2: -------------------------------------------------------------------------------- 1 | # Molecule managed 2 | 3 | {% if item.registry is defined %} 4 | FROM {{ item.registry.url }}/{{ item.image }} 5 | {% else %} 6 | FROM {{ item.image }} 7 | {% endif %} 8 | 9 | RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ 10 | elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python2-dnf bash && dnf clean all; \ 11 | elif [ $(command -v yum) ]; then yum makecache fast && yum update -y && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ 12 | elif [ $(command -v zypper) ]; then zypper refresh && zypper update -y && zypper install -y python sudo bash python-xml && zypper clean -a; \ 13 | elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ 14 | elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi 15 | -------------------------------------------------------------------------------- /molecule/default/INSTALL.rst: -------------------------------------------------------------------------------- 1 | ******* 2 | Install 3 | ******* 4 | 5 | Requirements 6 | ============ 7 | 8 | * Docker Engine 9 | * docker-py 10 | 11 | Install 12 | ======= 13 | 14 | .. code-block:: bash 15 | 16 | $ sudo pip install docker-py 17 | -------------------------------------------------------------------------------- /molecule/default/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | dependency: 4 | name: 'galaxy' 5 | driver: 6 | name: 'docker' 7 | lint: 8 | name: 'yamllint' 9 | platforms: 10 | - name: "airflow-docker-stretch" 11 | image: 'minimum2scp/systemd-stretch:latest' 12 | command: '/sbin/init' 13 | cap_add: 14 | - 'SYS_PTRACE' 15 | privileged: True 16 | - name: "airflow-docker-xenial" 17 | image: 'solita/ubuntu-systemd:16.04' 18 | command: '/sbin/init' 19 | cap_add: 20 | - 'SYS_PTRACE' 21 | privileged: True 22 | - name: "airflow-docker-bionic" 23 | image: 'solita/ubuntu-systemd:18.04' 24 | command: '/sbin/init' 25 | cap_add: 26 | - 'SYS_PTRACE' 27 | privileged: True 28 | provisioner: 29 | name: 'ansible' 30 | lint: 31 | name: 'ansible-lint' 32 | inventory: 33 | host_vars: 34 | airflow-docker-stretch: 35 | __airflow_system_dependencies: 36 | - name: 'default-libmysqlclient-dev' 37 | group_vars: 38 | all: 39 | airflow_database_engine: 'sqlite' 40 | airflow_extra_packages: 41 | - name: 'apache-airflow[crypto]' 42 | - name: 'apache-airflow[mysql]' 43 | - name: 'apache-airflow[password]' 44 | __airflow_system_dependencies: 45 | - name: 'libmysqlclient-dev' 46 | airflow_system_dependencies: "{{ 47 | _airflow_system_dependencies + __airflow_system_dependencies }}" 48 | airflow_webserver_hostname: '127.0.0.1' 49 | airflow_do_upgrade_db: False 50 | airflow_connections: 51 | - conn_id: 'foo' 52 | conn_uri: 'foobar://foo_user:foo_password@bar.foobar' 53 | airflow_variables: 54 | - key: 'my_key' 55 | value: 'my_key_value' 56 | - key: 'my_key_2' 57 | value: 'my_key_2_value' 58 | airflow_user_config: 59 | core: 60 | fernet_key: '3hw5_gCG2QGFn1zmLJdj0zfDTxoJS-1llvxzmxS3DHg=' 61 | scenario: 62 | name: 'default' 63 | verifier: 64 | name: 'testinfra' 65 | additional_files_or_dirs: 66 | - '../../tests' 67 | lint: 68 | name: 'flake8' 69 | options: 70 | sudo: True 71 | -------------------------------------------------------------------------------- /molecule/default/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Role tests 4 | 5 | - name: 'Converge | Prerequisites tasks' 6 | hosts: 'all' 7 | gather_facts: False 8 | tasks: 9 | - name: 'PREREQUISITES | APT | Do an apt-get update' 10 | become: True 11 | raw: 'apt-get update -qq' 12 | changed_when: False 13 | - name: 'PREREQUISITES | APT | Install python 2.7, sqlite3, iproute and net-tools' 14 | become: True 15 | raw: "apt-get install -qq python2.7 sqlite3 iproute{{ (ansible_host == 'airflow-docker-bionic') | ternary('2', '') }} net-tools" 16 | changed_when: False 17 | 18 | 19 | - name: 'Converge | Main tasks' 20 | hosts: 'all' 21 | pre_tasks: 22 | - block: 23 | - name: 'PREREQUISITES | APT | Install locales and git' 24 | apt: 25 | name: "{{ item }}" 26 | changed_when: False 27 | with_items: 28 | - 'git' 29 | - 'locales' 30 | - name: 'PREREQUISITES | Prepare locales management - Ubuntu' 31 | copy: 32 | dest: '/etc/locale.conf' 33 | content: 'LANG=en_US.UTF-8' 34 | changed_when: False 35 | - name: 'PREREQUISITES | Prepare locales management - Debian' 36 | copy: 37 | dest: '/etc/locale.gen' 38 | content: "en_US.UTF-8 UTF-8\n" 39 | changed_when: False 40 | - name: 'PREREQUISITES | Update locales' 41 | command: 'locale-gen' 42 | changed_when: False 43 | when: "(ansible_distribution_release | lower) != 'trusty'" 44 | - name: 'PREREQUISITES | Update locales' 45 | command: 'locale-gen en_US.UTF-8' 46 | changed_when: False 47 | when: "(ansible_distribution_release | lower) == 'trusty'" 48 | become: True 49 | roles: 50 | - role: "ansible-role-airflow" 51 | environment: 52 | LC_ALL: 'en_US.UTF-8' 53 | -------------------------------------------------------------------------------- /molecule/default/tests/test_installation.py: -------------------------------------------------------------------------------- 1 | """ 2 | Role tests 3 | """ 4 | 5 | import os 6 | import pytest 7 | from testinfra.utils.ansible_runner import AnsibleRunner 8 | 9 | testinfra_hosts = AnsibleRunner( 10 | os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') 11 | 12 | 13 | def test_airflow_user(host): 14 | """ 15 | Tests about airflow user configuration 16 | """ 17 | airflow_user = host.user('airflow') 18 | 19 | assert airflow_user.exists is True 20 | assert airflow_user.group == 'airflow' 21 | assert airflow_user.home == '/var/lib/airflow' 22 | assert airflow_user.shell == '/bin/false' 23 | 24 | 25 | def test_airflow_group(host): 26 | """ 27 | Tests about airflow group configuration 28 | """ 29 | 30 | assert host.group('airflow').exists is True 31 | 32 | 33 | @pytest.mark.parametrize('name,codenames', [ 34 | ('python3-dev', None), 35 | ('libpq-dev', None), 36 | ('libssl-dev', None), 37 | ('libffi-dev', None), 38 | ('build-essential', None), 39 | ('python-virtualenv', None), 40 | ('python-pip', None), 41 | ]) 42 | def test_prerequisites_packages(host, name, codenames): 43 | """ 44 | Tests about airflow prerequisites packages 45 | """ 46 | 47 | if host.system_info.distribution not in ['debian', 'ubuntu']: 48 | pytest.skip('{} ({}) distribution not managed'.format( 49 | host.system_info.distribution, host.system_info.release)) 50 | 51 | if codenames and host.system_info.codename.lower() not in codenames: 52 | pytest.skip('{} package not used with {} ({})'.format( 53 | name, host.system_info.distribution, host.system_info.codename)) 54 | 55 | assert host.package(name).is_installed 56 | 57 | 58 | def test_airflow_processes(host): 59 | """ 60 | Test about airflow processes 61 | """ 62 | 63 | assert len(host.process.filter(user='airflow', comm='airflow')) >= 2 64 | 65 | 66 | @pytest.mark.parametrize('name', [ 67 | ('airflow-webserver'), 68 | ('airflow-scheduler'), 69 | ]) 70 | def test_airflow_services(host, name): 71 | """ 72 | Test about airflow services 73 | """ 74 | 75 | if host.system_info.codename == 'jessie': 76 | output = host.check_output('service {} status'.format(name)) 77 | assert '{} running'.format(name) in output 78 | else: 79 | service = host.service(name) 80 | assert service.is_running 81 | assert service.is_enabled 82 | 83 | 84 | @pytest.mark.parametrize('path,path_type,user,group,mode', [ 85 | ('/var/run/airflow', 'directory', 'airflow', 'airflow', 0o700), 86 | ('/var/lib/airflow/airflow', 'directory', 'airflow', 'airflow', 0o700), 87 | ('/var/lib/airflow/venv', 'directory', 'airflow', 'airflow', 0o755), 88 | ( 89 | '/var/lib/airflow/airflow/airflow.cfg', 90 | 'file', 'airflow', 'airflow', 0o400 91 | ), 92 | ]) 93 | def test_airflow_paths(host, path, path_type, user, group, mode): 94 | """ 95 | Tests Airflow paths 96 | """ 97 | 98 | current_path = host.file(path) 99 | 100 | assert current_path.exists 101 | 102 | if path_type == 'directory': 103 | assert current_path.is_directory 104 | elif path_type == 'file': 105 | assert current_path.is_file 106 | assert current_path.user == user 107 | assert current_path.group == group 108 | assert current_path.mode == mode 109 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | docker==4.2.2 2 | molecule==2.15.0 3 | pytest==5.4.3 4 | python-vagrant==0.5.15 5 | tox==3.16.1 6 | -------------------------------------------------------------------------------- /tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Main tasks file for airflow role 4 | 5 | - name: 'INIT | Manage variables to use for our target' 6 | import_tasks: "{{ role_path }}/tasks/manage_variables.yml" 7 | tags: 8 | - 'role::airflow' 9 | - 'role::airflow::init' 10 | - 'role::airflow::install' 11 | 12 | 13 | # Installation tasks management 14 | - name: 'INSTALL | Include installation tasks' 15 | import_tasks: "{{ role_path }}/tasks/manage_installation.yml" 16 | environment: 17 | AIRFLOW_HOME: '{{ airflow_config.core.airflow_home }}' 18 | become: True 19 | tags: 20 | - 'role::airflow' 21 | - 'role::airflow::install' 22 | 23 | 24 | # Configuration tasks management 25 | - name: 'CONFIG | Include configuration tasks' 26 | import_tasks: "{{ role_path }}/tasks/manage_configuration.yml" 27 | environment: 28 | AIRFLOW_HOME: '{{ airflow_config.core.airflow_home }}' 29 | become: True 30 | tags: 31 | - 'role::airflow' 32 | - 'role::airflow::config' 33 | 34 | 35 | # Services tasks management 36 | - name: 'SERVICE | Include services configuration tasks' 37 | import_tasks: "{{ role_path }}/tasks/manage_services.yml" 38 | become: True 39 | tags: 40 | - 'role::airflow' 41 | - 'role::airflow::config' 42 | - 'role::airflow::install' 43 | 44 | 45 | # Connections tasks management 46 | - name: 'CONFIG | Include connections tasks' 47 | import_tasks: "{{ role_path }}/tasks/manage_connections.yml" 48 | environment: 49 | AIRFLOW_HOME: '{{ airflow_config.core.airflow_home }}' 50 | become: True 51 | tags: 52 | - 'role::airflow' 53 | - 'role::airflow::config' 54 | 55 | 56 | # Airflow variables tasks management 57 | - name: 'CONFIG | Include connections tasks' 58 | import_tasks: "{{ role_path }}/tasks/manage_airflow_variables.yml" 59 | environment: 60 | AIRFLOW_HOME: '{{ airflow_config.core.airflow_home }}' 61 | become: True 62 | tags: 63 | - 'role::airflow' 64 | - 'role::airflow::config' 65 | -------------------------------------------------------------------------------- /tasks/manage_airflow_variables.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Manage variables using Airflow CLI 4 | 5 | - name: 'CONFIG | VARIABLES | Get existing variables' 6 | become_user: "{{ airflow_user_name }}" 7 | shell: > 8 | {{ airflow_virtualenv }}/bin/airflow variables | grep -vE '^\[' 9 | changed_when: false 10 | register: 'airflow_existing_variables' 11 | 12 | 13 | # Only add and delete actions are managed, not update 14 | - name: 'CONFIG | VARIABLES | Remove existing variables' 15 | shell: > 16 | {{ airflow_virtualenv }}/bin/airflow variables --delete {{ item }} 17 | changed_when: false 18 | with_items: "{{ airflow_existing_variables.stdout_lines }}" 19 | when: 'airflow_drop_existing_variables_before_add | bool' 20 | 21 | 22 | - name: 'CONFIG | VARIABLES | Add variables' 23 | shell: > 24 | {{ airflow_virtualenv }}/bin/airflow variables --set {{ item.key }} \ 25 | {{ item.value }} 26 | changed_when: false 27 | no_log: True 28 | with_items: "{{ airflow_variables }}" 29 | -------------------------------------------------------------------------------- /tasks/manage_configuration.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Airflow configuration tasks 4 | 5 | # Manage configuration templating 6 | - name: 'CONFIG | Manage configuration templating' 7 | become_user: "{{ airflow_user_name }}" 8 | template: 9 | src: "{{ role_path }}/templates/airflow.cfg.j2" 10 | dest: "{{ airflow_config.core.airflow_home }}/airflow.cfg" 11 | owner: "{{ airflow_user_name }}" 12 | group: "{{ airflow_user_group }}" 13 | mode: '0400' 14 | notify: 'Restart airflow' 15 | 16 | 17 | # Manage logs folder 18 | - name: 'CONFIG | Manage log folder' 19 | file: 20 | path: "{{ airflow_log_path }}" 21 | owner: "{{ airflow_log_owner }}" 22 | group: "{{ airflow_log_group }}" 23 | mode: "{{ airflow_log_mode }}" 24 | state: 'directory' 25 | recurse: yes 26 | 27 | 28 | # Manage pid folder 29 | - name: 'CONFIG | Manage pid folder' 30 | file: 31 | path: "{{ airflow_pid_path }}" 32 | owner: "{{ airflow_pid_owner }}" 33 | group: "{{ airflow_pid_group }}" 34 | mode: "{{ airflow_pid_mode }}" 35 | state: 'directory' 36 | recurse: yes 37 | 38 | 39 | # Manage logrotate configuration 40 | - name: 'CONFIG | Manage logrotate configuration' 41 | template: 42 | src: "{{ role_path }}/templates/logrotate.j2" 43 | dest: "{{ item.filename }}" 44 | owner: 'root' 45 | group: 'root' 46 | mode: '0644' 47 | with_items: "{{ airflow_logrotate_config }}" 48 | 49 | 50 | # Manage database initialize if first deployment 51 | - name: 'CONFIG | Manage database initialize' 52 | become_user: "{{ airflow_user_name }}" 53 | command: "{{ airflow_virtualenv }}/bin/airflow initdb" 54 | when: 55 | - "not airflow_config_stat.stat.exists" 56 | - "airflow_do_init_db | bool" 57 | 58 | 59 | # Set Sqlite backend permissions 60 | - name: 'CONFIG | Ensure SQLite backend file has good permissions' 61 | file: 62 | path: "{{ airflow_database_sqlite_file_path }}" 63 | owner: "{{ airflow_user_name }}" 64 | group: "{{ airflow_user_group }}" 65 | state: 'file' 66 | when: 67 | - "airflow_database_engine == 'sqlite'" 68 | 69 | 70 | # Manage database upgrade if not first deployment 71 | - name: 'CONFIG | Manage database upgrade' 72 | become_user: "{{ airflow_user_name }}" 73 | command: "{{ airflow_virtualenv }}/bin/airflow upgradedb" 74 | when: 75 | - "airflow_config_stat.stat.exists" 76 | - "airflow_do_upgrade_db | bool" 77 | -------------------------------------------------------------------------------- /tasks/manage_connections.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Manage connections using Airflow CLI 4 | 5 | - name: 'CONFIG | CONNECTIONS | Get existing connections' 6 | become_user: "{{ airflow_user_name }}" 7 | shell: > 8 | {{ airflow_virtualenv }}/bin/airflow connections --list \ 9 | | grep -E "\s+'" | cut -d "'" -f2 10 | changed_when: false 11 | failed_when: False 12 | register: 'airflow_existing_connections' 13 | 14 | 15 | # Only add and delete actions are managed, not update 16 | - name: 'CONFIG | CONNECTIONS | Remove existing connections' 17 | shell: > 18 | {{ airflow_virtualenv }}/bin/airflow connections --delete \ 19 | --conn_id {{ item }} 20 | changed_when: false 21 | with_items: "{{ airflow_existing_connections.stdout_lines }}" 22 | when: 'airflow_drop_existing_connections_before_add | bool' 23 | 24 | 25 | - name: 'CONFIG | CONNECTIONS | Add connections without extra settings' 26 | shell: > 27 | {{ airflow_virtualenv }}/bin/airflow connections --add \ 28 | --conn_id {{ item.conn_id }} --conn_uri {{ item.conn_uri }} 29 | changed_when: false 30 | no_log: True 31 | with_items: "{{ airflow_connections }}" 32 | when: "(item.conn_extra | default('')) == ''" 33 | 34 | 35 | - name: 'CONFIG | CONNECTIONS | Add connections with extra settings' 36 | shell: > 37 | {{ airflow_virtualenv }}/bin/airflow connections --add \ 38 | --conn_id {{ item.conn_id }} --conn_uri {{ item.conn_uri }} \ 39 | --conn_extra {{ item.conn_extra | quote }} 40 | changed_when: false 41 | no_log: True 42 | with_items: "{{ airflow_connections }}" 43 | when: "(item.conn_extra | default('')) != ''" 44 | -------------------------------------------------------------------------------- /tasks/manage_installation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Airflow installation tasks 4 | 5 | # Manage system dependencies 6 | - name: 'INSTALL | Manage system dependencies' 7 | package: 8 | name: "{{ item.name }}" 9 | state: "{{ item.state | default('present') }}" 10 | with_items: "{{ airflow_system_dependencies }}" 11 | 12 | 13 | # Create user 14 | - name: 'INSTALL | Manage airflow user creation' 15 | user: 16 | name: "{{ airflow_user_name }}" 17 | createhome: True 18 | home: "{{ airflow_user_home_path}}" 19 | state: 'present' 20 | shell: "{{ airflow_user_shell }}" 21 | 22 | 23 | # Check if configuration file exists 24 | # This has to happen before the install, because the install creates a default file in $AIRFLOW_HOME 25 | - name: 'INSTALL | Check if configuration file exists' 26 | stat: 27 | path: "{{ airflow_config.core.airflow_home }}/airflow.cfg" 28 | register: 'airflow_config_stat' 29 | changed_when: False 30 | 31 | 32 | - name: 'INSTALL | Manage permissions on user home directory' 33 | file: 34 | path: "{{ airflow_user_home_path }}" 35 | owner: "{{ airflow_user_name }}" 36 | group: "{{ airflow_user_name }}" 37 | mode: "{{ airflow_user_home_mode }}" 38 | state: directory 39 | 40 | 41 | - name: 'INSTALL | Manage permissions on airflow venv directory' 42 | file: 43 | path: "{{ airflow_virtualenv }}" 44 | owner: "{{ airflow_user_name }}" 45 | group: "{{ airflow_user_name }}" 46 | mode: "{{ airflow_virtualenv_mode }}" 47 | state: directory 48 | 49 | - name: 'INSTALL | Manage permissions on airflow home directory' 50 | file: 51 | path: "{{ airflow_config.core.airflow_home }}" 52 | owner: "{{ airflow_user_name }}" 53 | group: "{{ airflow_user_name }}" 54 | mode: "{{ airflow_config.core.airflow_home_mode }}" 55 | state: directory 56 | 57 | 58 | - name: 'INSTALL | Manage permissions on airflow dags directory' 59 | file: 60 | path: "{{ airflow_config.core.dags_folder }}" 61 | owner: "{{ airflow_user_name }}" 62 | group: "{{ airflow_user_name }}" 63 | mode: "{{ airflow_config.core.dags_folder_mode }}" 64 | state: directory 65 | 66 | 67 | # setup environment for all users, AIRFLOW_HOME, etc 68 | - name: 'INSTALL | Manage /etc/profile.d/ file' 69 | template: 70 | src: "{{ role_path }}/templates/airflow.profile.j2" 71 | dest: "/etc/profile.d/airflow.sh" 72 | when: airflow_provide_user_env 73 | 74 | 75 | # Install airflow packages 76 | - name: 'INSTALL | Manage airflow installation' 77 | become_user: "{{ airflow_user_name }}" 78 | pip: 79 | name: "{{ item.name }}" 80 | version: "{{ item.version | default(omit) }}" 81 | state: "{{ item.state | default('present') }}" 82 | virtualenv: "{{ airflow_virtualenv }}" 83 | virtualenv_python: "{{ airflow_python_version }}" 84 | changed_when: "airflow_pip_changed_when" 85 | with_items: "{{ airflow_packages }}" 86 | when: (not airflow_pip_custom_version_install) or 87 | (item.version is not defined) or 88 | ( (not item.version | search('<') ) and (not item.version | search('>') ) ) 89 | 90 | 91 | # Install airflow packages 92 | # Ansible's pip module doesn't currently support complex version strings 93 | # https://github.com/ansible/ansible/issues/19321 94 | - name: 'INSTALL | Manage airflow installation - complex pip versions' 95 | become_user: "{{ airflow_user_name }}" 96 | command: "{{ airflow_virtualenv }}/bin/pip install '{{ item.name }}{{ item.version }}'" 97 | changed_when: "airflow_pip_changed_when" 98 | with_items: "{{ airflow_packages }}" 99 | when: (airflow_pip_custom_version_install) and 100 | (item.version is defined) and 101 | ( (item.version | search('<') ) or (item.version | search('>') ) ) 102 | 103 | 104 | # Install airflow extra packages 105 | - name: 'INSTALL | Manage airflow extra packages installation' 106 | become_user: "{{ airflow_user_name }}" 107 | pip: 108 | name: "{{ item.name }}" 109 | state: "{{ item.state | default('present') }}" 110 | virtualenv: "{{ airflow_virtualenv }}" 111 | virtualenv_python: "{{ airflow_python_version }}" 112 | changed_when: "airflow_pip_changed_when" 113 | with_items: "{{ airflow_extra_packages }}" 114 | -------------------------------------------------------------------------------- /tasks/manage_services.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Airflow services management 4 | 5 | - name: 'SERVICE | Manage environment file' 6 | template: 7 | src: "{{ role_path }}/templates/airflow.env.j2" 8 | dest: "{{ airflow_paths.files.environment.path }}" 9 | owner: "{{ airflow_paths.files.environment.owner | default('root') }}" 10 | group: "{{ airflow_paths.files.environment.group | default('root') }}" 11 | mode: "{{ airflow_paths.files.environment.mode | default('0644') }}" 12 | notify: 13 | - 'Restart airflow-webserver' 14 | - 'Restart airflow-scheduler' 15 | 16 | 17 | - name: 'SERVICE | Manage systemd services' 18 | template: 19 | src: "{{ role_path }}/templates/systemd.service.j2" 20 | dest: "{{ item.dest }}" 21 | owner: "{{ item.owner | default('root') }}" 22 | group: "{{ item.group | default('root') }}" 23 | mode: "{{ item.mode | default('0644') }}" 24 | notify: "{{ item.handler }}" 25 | register: airflow_systemd_updated 26 | with_items: "{{ airflow_services_systemd }}" 27 | when: "is_systemd_managed_system | bool" 28 | 29 | - name: 'SERVICE | systemd daemon-reload' 30 | systemd: 31 | daemon_reload: true 32 | when: "(is_systemd_managed_system | bool) and (airflow_systemd_updated.changed | bool)" 33 | 34 | 35 | # Services states 36 | - name: 'SERVICE | Manage airflow services states' 37 | service: 38 | name: "{{ item.name }}" 39 | enabled: "{{ item.enabled | default(False) }}" 40 | state: "{{ item.state }}" 41 | with_items: "{{ airflow_services_states }}" 42 | -------------------------------------------------------------------------------- /tasks/manage_variables.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # All tasks about variables management 4 | 5 | - name: 'INIT | VARIABLES | Check if OS family vars file exists' 6 | become: False 7 | stat: 8 | path: "{{ role_path }}/vars/os_family/{{ ansible_os_family | lower }}.yml" 9 | register: 'airflow_check_os_family_vars' 10 | delegate_to: '127.0.0.1' 11 | 12 | 13 | - name: 'INIT | VARIABLES | Check if OS distribution vars file exists' 14 | become: False 15 | stat: 16 | path: "{{ role_path }}/vars/os_distribution/{{ ansible_distribution | lower }}.yml" 17 | register: 'airflow_check_os_distribution_vars' 18 | delegate_to: '127.0.0.1' 19 | 20 | 21 | - name: 'INIT | VARIABLES | Check if OS release vars file exists' 22 | become: False 23 | stat: 24 | path: "{{ role_path }}/vars/os_distribution/{{ ansible_distribution | lower }}/{{ ansible_distribution_release | lower }}.yml" 25 | register: 'airflow_check_os_release_vars' 26 | delegate_to: '127.0.0.1' 27 | 28 | 29 | - name: 'INIT | VARIABLES | Load OS family vars file' 30 | include_vars: "{{ role_path }}/vars/os_family/{{ ansible_os_family | lower }}.yml" 31 | when: "airflow_check_os_family_vars.stat.exists" 32 | 33 | 34 | - name: 'INIT | VARIABLES | Load OS distribution vars file' 35 | include_vars: "{{ role_path }}/vars/os_distribution/{{ ansible_distribution | lower }}.yml" 36 | when: "airflow_check_os_distribution_vars.stat.exists" 37 | 38 | 39 | - name: 'INIT | VARIABLES | Load OS release vars file' 40 | include_vars: "{{ role_path }}/vars/os_distribution/{{ ansible_distribution | lower }}/{{ ansible_distribution_release | lower }}.yml" 41 | when: "airflow_check_os_release_vars.stat.exists" 42 | -------------------------------------------------------------------------------- /templates/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infOpen/ansible-role-airflow/23c6cf0f465fe34df6b0b12e60d344abbfdf4260/templates/.keep -------------------------------------------------------------------------------- /templates/airflow.cfg.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for section_name, section_items in airflow_config.iteritems() %} 4 | 5 | [{{ section_name }}] 6 | {% for item_name, item_value in section_items.iteritems() %} 7 | {{ item_name }} = {{ item_value }} 8 | {% endfor %} 9 | {% endfor %} 10 | -------------------------------------------------------------------------------- /templates/airflow.env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | # General 4 | AIRFLOW_HOME="{{ airflow_config.core.airflow_home }}" 5 | AIRFLOW_CONFIG="{{ airflow_config.core.airflow_home }}/airflow.cfg" 6 | AIRFLOW_BINARY_PATH="{{ airflow_virtualenv ~ '/bin' }}" 7 | 8 | # Webserver specific 9 | {% if airflow_config.webserver.debug %} 10 | AIRFLOW_WEBSERVER_DEBUG="--debug" 11 | {% endif %} 12 | AIRFLOW_WEBSERVER_HOSTNAME="{{ airflow_config.webserver.hostname }}" 13 | AIRFLOW_WEBSERVER_LOG_FILE="{{ airflow_config.webserver.log_file }}" 14 | AIRFLOW_WEBSERVER_PID_FILE="{{ airflow_config.webserver.pid }}" 15 | AIRFLOW_WEBSERVER_PORT="{{ airflow_config.webserver.web_server_port }}" 16 | AIRFLOW_WEBSERVER_WORKER_CLASS="{{ airflow_config.webserver.worker_class }}" 17 | AIRFLOW_WEBSERVER_WORKER_TIMEOUT="{{ airflow_config.webserver.worker_timeout }}" 18 | AIRFLOW_WEBSERVER_WORKERS="{{ airflow_config.webserver.workers }}" 19 | 20 | # Scheduler specific 21 | AIRFLOW_SCHEDULER_LOG_FILE="{{ airflow_config.scheduler.log_file }}" 22 | AIRFLOW_SCHEDULER_PID_FILE="{{ airflow_config.scheduler.pid }}" 23 | AIRFLOW_SCHEDULER_RUNS="{{ airflow_config.scheduler.num_runs }}" 24 | -------------------------------------------------------------------------------- /templates/airflow.profile.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | # General 4 | export AIRFLOW_HOME="{{ airflow_config.core.airflow_home }}" 5 | export PATH="$PATH:{{ airflow_virtualenv ~ '/bin' }}" 6 | -------------------------------------------------------------------------------- /templates/logrotate.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {{ item.log_pattern | join(' ') }} { 4 | {% for option in item.options %} 5 | {{ option }} 6 | {% endfor %} 7 | } 8 | -------------------------------------------------------------------------------- /templates/systemd.service.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for section in item.options.keys() %} 4 | 5 | [{{ section }}] 6 | {% for key, value in item.options[section].iteritems() %} 7 | {{ key }}={{ value }} 8 | {% endfor %} 9 | {% endfor %} 10 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infOpen/ansible-role-airflow/23c6cf0f465fe34df6b0b12e60d344abbfdf4260/tests/__init__.py -------------------------------------------------------------------------------- /tests/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Role tests 4 | 5 | - hosts: 'airflow-vagrant-xenial64:airflow-docker-xenial' 6 | name: 'Install tests prerequisites' 7 | gather_facts: False 8 | tasks: 9 | - name: 'PREREQUISITES | APT | Do an apt-get update' 10 | become: True 11 | raw: 'apt-get update -qq' 12 | changed_when: False 13 | - name: 'PREREQUISITES | APT | Install python 2.7, iproute and net-tools' 14 | become: True 15 | raw: 'apt-get install -qq python2.7 iproute net-tools' 16 | changed_when: False 17 | 18 | 19 | - hosts: 'all' 20 | roles: 21 | - role: "{{ role_name }}" 22 | vars: 23 | role_name: "{{ playbook_dir | basename }}" 24 | -------------------------------------------------------------------------------- /tests/test_filter_plugins.py: -------------------------------------------------------------------------------- 1 | """ 2 | Fake test for plugins filters 3 | """ 4 | 5 | 6 | def test_fake(): 7 | assert True 8 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | minversion = 1.8 3 | envlist = py{27}-ansible{24,25} 4 | skipsdist = true 5 | 6 | [testenv] 7 | passenv = * 8 | deps = 9 | -rrequirements.txt 10 | ansible24: ansible>=2.4,<2.5 11 | ansible25: ansible>=2.5,<2.6 12 | commands = 13 | molecule --debug test 14 | -------------------------------------------------------------------------------- /vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Main vars file for airflow role 4 | -------------------------------------------------------------------------------- /vars/os_distribution/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infOpen/ansible-role-airflow/23c6cf0f465fe34df6b0b12e60d344abbfdf4260/vars/os_distribution/.keep -------------------------------------------------------------------------------- /vars/os_distribution/debian/stretch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Debian Stretch specific vars 4 | 5 | _airflow_python_version: 'python3.5' 6 | 7 | # Services management 8 | _is_systemd_managed_system: True 9 | -------------------------------------------------------------------------------- /vars/os_distribution/ubuntu/bionic.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Ubuntu Bionic specific vars 4 | 5 | 6 | # Python 7 | _airflow_python_version: 'python3.6' 8 | 9 | # Services management 10 | _is_systemd_managed_system: True 11 | -------------------------------------------------------------------------------- /vars/os_distribution/ubuntu/xenial.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Ubuntu Xenial specific vars 4 | 5 | 6 | # Python 7 | _airflow_python_version: 'python3.5' 8 | 9 | # Services management 10 | _is_systemd_managed_system: True 11 | -------------------------------------------------------------------------------- /vars/os_family/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infOpen/ansible-role-airflow/23c6cf0f465fe34df6b0b12e60d344abbfdf4260/vars/os_family/.keep -------------------------------------------------------------------------------- /vars/os_family/debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Debian OS family specific vars 4 | 5 | _airflow_system_dependencies: 6 | - name: 'python3-dev' 7 | - name: 'libpq-dev' 8 | - name: 'libssl-dev' 9 | - name: 'libffi-dev' 10 | - name: 'libxml2' 11 | - name: 'libxml2-dev' 12 | - name: 'libxslt1-dev' 13 | - name: 'build-essential' 14 | - name: 'python-virtualenv' 15 | - name: 'python-pip' 16 | --------------------------------------------------------------------------------