├── .dockerignore ├── .gitignore ├── .vscode └── settings.json ├── Dockerfile ├── LICENSE ├── README.md ├── assets └── images │ ├── spark_submit_dag.png │ └── spark_submit_design.png ├── config └── airflow.cfg ├── dags ├── scripts │ └── spark │ │ └── random_text_classification.py └── spark_submit_airflow.py ├── docker-compose-LocalExecutor.yml ├── requirements.txt ├── script └── entrypoint.sh └── workflows └── ci.yml /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### Vim ### 2 | [._]*.s[a-w][a-z] 3 | [._]s[a-w][a-z] 4 | *.un~ 5 | Session.vim 6 | .netrwhist 7 | *~ 8 | 9 | ### SublimeText ### 10 | # cache files for sublime text 11 | *.tmlanguage.cache 12 | *.tmPreferences.cache 13 | *.stTheme.cache 14 | 15 | # workspace files are user-specific 16 | *.sublime-workspace 17 | 18 | # project files should be checked into the repository, unless a significant 19 | # proportion of contributors will probably not be using SublimeText 20 | # *.sublime-project 21 | 22 | # sftp configuration file 23 | sftp-config.json 24 | 25 | # Python 26 | __pycache__ 27 | 28 | # datasets 29 | *movie_review.csv* -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.pythonPath": "/usr/local/opt/python/bin/python3.7" 3 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # VERSION 1.10.9 2 | # AUTHOR: Matthieu "Puckel_" Roisil 3 | # DESCRIPTION: Basic Airflow container 4 | # BUILD: docker build --rm -t puckel/docker-airflow . 5 | # SOURCE: https://github.com/puckel/docker-airflow 6 | 7 | FROM python:3.7-slim-buster 8 | LABEL maintainer="Puckel_" 9 | 10 | # Never prompt the user for choices on installation/configuration of packages 11 | ENV DEBIAN_FRONTEND noninteractive 12 | ENV TERM linux 13 | 14 | # Airflow 15 | ARG AIRFLOW_VERSION=1.10.9 16 | ARG AIRFLOW_USER_HOME=/usr/local/airflow 17 | ARG AIRFLOW_DEPS="" 18 | ARG PYTHON_DEPS="" 19 | ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} 20 | 21 | # Define en_US. 22 | ENV LANGUAGE en_US.UTF-8 23 | ENV LANG en_US.UTF-8 24 | ENV LC_ALL en_US.UTF-8 25 | ENV LC_CTYPE en_US.UTF-8 26 | ENV LC_MESSAGES en_US.UTF-8 27 | 28 | # Disable noisy "Handling signal" log messages: 29 | # ENV GUNICORN_CMD_ARGS --log-level WARNING 30 | 31 | RUN set -ex \ 32 | && buildDeps=' \ 33 | freetds-dev \ 34 | libkrb5-dev \ 35 | libsasl2-dev \ 36 | libssl-dev \ 37 | libffi-dev \ 38 | libpq-dev \ 39 | git \ 40 | ' \ 41 | && apt-get update -yqq \ 42 | && apt-get upgrade -yqq \ 43 | && apt-get install -yqq --no-install-recommends \ 44 | $buildDeps \ 45 | freetds-bin \ 46 | build-essential \ 47 | default-libmysqlclient-dev \ 48 | apt-utils \ 49 | curl \ 50 | rsync \ 51 | netcat \ 52 | locales \ 53 | && sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \ 54 | && locale-gen \ 55 | && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ 56 | && useradd -ms /bin/bash -d ${AIRFLOW_USER_HOME} airflow \ 57 | && pip install -U pip setuptools wheel \ 58 | && pip install pytz \ 59 | && pip install pyOpenSSL \ 60 | && pip install ndg-httpsclient \ 61 | && pip install pyasn1 \ 62 | && pip install apache-airflow[crypto,celery,postgres,hive,jdbc,mysql,ssh${AIRFLOW_DEPS:+,}${AIRFLOW_DEPS}]==${AIRFLOW_VERSION} \ 63 | && pip install 'redis==3.2' \ 64 | && if [ -n "${PYTHON_DEPS}" ]; then pip install ${PYTHON_DEPS}; fi \ 65 | && apt-get purge --auto-remove -yqq $buildDeps \ 66 | && apt-get autoremove -yqq --purge \ 67 | && apt-get clean \ 68 | && rm -rf \ 69 | /var/lib/apt/lists/* \ 70 | /tmp/* \ 71 | /var/tmp/* \ 72 | /usr/share/man \ 73 | /usr/share/doc \ 74 | /usr/share/doc-base 75 | 76 | COPY script/entrypoint.sh /entrypoint.sh 77 | COPY config/airflow.cfg ${AIRFLOW_USER_HOME}/airflow.cfg 78 | 79 | RUN chown -R airflow: ${AIRFLOW_USER_HOME} 80 | 81 | EXPOSE 8080 5555 8793 82 | 83 | USER airflow 84 | WORKDIR ${AIRFLOW_USER_HOME} 85 | ENTRYPOINT ["/entrypoint.sh"] 86 | CMD ["webserver"] 87 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2017 Matthieu "Puckel_" Roisil 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # How to submit Spark jobs to EMR cluster from Airflow 2 | 3 | This is the repository for blog at [How to submit Spark jobs to EMR cluster from Airflow](http://startdataengineering.com/post/how-to-submit-spark-jobs-to-emr-cluster-from-airflow). 4 | 5 | # Prerequisites 6 | 7 | 1. [docker](https://docs.docker.com/get-docker/) (make sure to have docker-compose as well). 8 | 2. [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) to clone the starter repo. 9 | 3. [AWS account](https://aws.amazon.com/) to set up required cloud services. 10 | 4. [Install](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) and [configure](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config) AWS CLI on your machine. 11 | 12 | # Design 13 | 14 | ![DAG Design](assets/images/spark_submit_design.png) 15 | 16 | # Data 17 | 18 | From the project directory do 19 | 20 | ```bash 21 | wget https://www.dropbox.com/sh/amdyc6z8744hrl5/AADS8aPTbA-dRAUjvVfjTo2qa/movie_review 22 | mkdir ./dags/data 23 | mv movie_review ./dags/data/movie_review.csv 24 | ``` 25 | 26 | # Setup and run 27 | 28 | If this is your first time using AWS, make sure to check for presence of the `EMR_EC2_DefaultRole` and `EMR_DefaultRole` default role as shown below. 29 | 30 | ```bash 31 | aws iam list-roles | grep 'EMR_DefaultRole\|EMR_EC2_DefaultRole' 32 | # "RoleName": "EMR_DefaultRole", 33 | # "RoleName": "EMR_EC2_DefaultRole", 34 | ``` 35 | 36 | If the roles not present, create them using the following command 37 | 38 | ```bash 39 | aws emr create-default-roles 40 | ``` 41 | 42 | Also create a bucket, using the following command. 43 | 44 | ```bash 45 | aws s3api create-bucket --acl public-read-write --bucket 46 | ``` 47 | 48 | Replace `` with your bucket name. eg.) if your bucket name is `my-bucket` then the above command becomes `aws s3api create-bucket --acl public-read-write --bucket my-bucket` 49 | 50 | and press `q` to exit the prompt 51 | 52 | After use, you can delete your S3 bucket as shown below 53 | 54 | ```bash 55 | aws s3api delete-bucket --bucket 56 | ``` 57 | 58 | and press `q` to exit the prompt 59 | 60 | ```bash 61 | docker-compose -f docker-compose-LocalExecutor.yml up -d 62 | ``` 63 | 64 | go to [http://localhost:8080/admin/](http://localhost:8080/admin/) and turn on the `spark_submit_airflow` DAG. You can check the status at [http://localhost:8080/admin/airflow/graph?dag_id=spark_submit_airflow](http://localhost:8080/admin/airflow/graph?dag_id=spark_submit_airflow). 65 | 66 | ![DAG](assets/images/spark_submit_dag.png) 67 | 68 | # Terminate local instance 69 | 70 | ```bash 71 | docker-compose -f docker-compose-LocalExecutor.yml down 72 | ``` 73 | 74 | ```bash 75 | aws s3api delete-bucket --bucket 76 | ``` 77 | 78 | # Contact 79 | 80 | website: https://www.startdataengineering.com/ 81 | 82 | twitter: https://twitter.com/start_data_eng 83 | -------------------------------------------------------------------------------- /assets/images/spark_submit_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/spark_submit_airflow/21408da9a10f70143e3d7d608ab261dead059f90/assets/images/spark_submit_dag.png -------------------------------------------------------------------------------- /assets/images/spark_submit_design.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/spark_submit_airflow/21408da9a10f70143e3d7d608ab261dead059f90/assets/images/spark_submit_design.png -------------------------------------------------------------------------------- /config/airflow.cfg: -------------------------------------------------------------------------------- 1 | [core] 2 | # The folder where your airflow pipelines live, most likely a 3 | # subfolder in a code repository. This path must be absolute. 4 | dags_folder = /usr/local/airflow/dags 5 | 6 | # The folder where airflow should store its log files 7 | # This path must be absolute 8 | base_log_folder = /usr/local/airflow/logs 9 | 10 | # Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. 11 | # Set this to True if you want to enable remote logging. 12 | remote_logging = False 13 | 14 | # Users must supply an Airflow connection id that provides access to the storage 15 | # location. 16 | remote_log_conn_id = 17 | remote_base_log_folder = 18 | encrypt_s3_logs = False 19 | 20 | # Logging level 21 | logging_level = INFO 22 | 23 | # Logging level for Flask-appbuilder UI 24 | fab_logging_level = WARN 25 | 26 | # Logging class 27 | # Specify the class that will specify the logging configuration 28 | # This class has to be on the python classpath 29 | # Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG 30 | logging_config_class = 31 | 32 | # Flag to enable/disable Colored logs in Console 33 | # Colour the logs when the controlling terminal is a TTY. 34 | colored_console_log = True 35 | 36 | # Log format for when Colored logs is enabled 37 | colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s 38 | colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter 39 | 40 | # Format of Log line 41 | log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s 42 | simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s 43 | 44 | # Log filename format 45 | log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log 46 | log_processor_filename_template = {{ filename }}.log 47 | dag_processor_manager_log_location = /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log 48 | 49 | # Name of handler to read task instance logs. 50 | # Default to use task handler. 51 | task_log_reader = task 52 | 53 | # Hostname by providing a path to a callable, which will resolve the hostname. 54 | # The format is "package:function". 55 | # 56 | # For example, default value "socket:getfqdn" means that result from getfqdn() of "socket" 57 | # package will be used as hostname. 58 | # 59 | # No argument should be required in the function specified. 60 | # If using IP address as hostname is preferred, use value ``airflow.utils.net:get_host_ip_address`` 61 | hostname_callable = socket:getfqdn 62 | 63 | # Default timezone in case supplied date times are naive 64 | # can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) 65 | default_timezone = utc 66 | 67 | # The executor class that airflow should use. Choices include 68 | # SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor, KubernetesExecutor 69 | executor = SequentialExecutor 70 | 71 | # The SqlAlchemy connection string to the metadata database. 72 | # SqlAlchemy supports many different database engine, more information 73 | # their website 74 | # sql_alchemy_conn = sqlite:////tmp/airflow.db 75 | 76 | # The encoding for the databases 77 | sql_engine_encoding = utf-8 78 | 79 | # If SqlAlchemy should pool database connections. 80 | sql_alchemy_pool_enabled = True 81 | 82 | # The SqlAlchemy pool size is the maximum number of database connections 83 | # in the pool. 0 indicates no limit. 84 | sql_alchemy_pool_size = 5 85 | 86 | # The maximum overflow size of the pool. 87 | # When the number of checked-out connections reaches the size set in pool_size, 88 | # additional connections will be returned up to this limit. 89 | # When those additional connections are returned to the pool, they are disconnected and discarded. 90 | # It follows then that the total number of simultaneous connections the pool will allow 91 | # is pool_size + max_overflow, 92 | # and the total number of "sleeping" connections the pool will allow is pool_size. 93 | # max_overflow can be set to -1 to indicate no overflow limit; 94 | # no limit will be placed on the total number of concurrent connections. Defaults to 10. 95 | sql_alchemy_max_overflow = 10 96 | 97 | # The SqlAlchemy pool recycle is the number of seconds a connection 98 | # can be idle in the pool before it is invalidated. This config does 99 | # not apply to sqlite. If the number of DB connections is ever exceeded, 100 | # a lower config value will allow the system to recover faster. 101 | sql_alchemy_pool_recycle = 1800 102 | 103 | # Check connection at the start of each connection pool checkout. 104 | # Typically, this is a simple statement like "SELECT 1". 105 | # More information here: 106 | # https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic 107 | sql_alchemy_pool_pre_ping = True 108 | 109 | # The schema to use for the metadata database. 110 | # SqlAlchemy supports databases with the concept of multiple schemas. 111 | sql_alchemy_schema = 112 | 113 | # The amount of parallelism as a setting to the executor. This defines 114 | # the max number of task instances that should run simultaneously 115 | # on this airflow installation 116 | parallelism = 1 117 | 118 | # The number of task instances allowed to run concurrently by the scheduler 119 | dag_concurrency = 1 120 | 121 | # Are DAGs paused by default at creation 122 | dags_are_paused_at_creation = True 123 | 124 | # The maximum number of active DAG runs per DAG 125 | max_active_runs_per_dag = 1 126 | 127 | # Whether to load the examples that ship with Airflow. It's good to 128 | # get started, but you probably want to set this to False in a production 129 | # environment 130 | load_examples = True 131 | 132 | # Where your Airflow plugins are stored 133 | plugins_folder = /usr/local/airflow/plugins 134 | 135 | # Secret key to save connection passwords in the db 136 | fernet_key = $FERNET_KEY 137 | 138 | # Whether to disable pickling dags 139 | donot_pickle = False 140 | 141 | # How long before timing out a python file import 142 | dagbag_import_timeout = 30 143 | 144 | # How long before timing out a DagFileProcessor, which processes a dag file 145 | dag_file_processor_timeout = 50 146 | 147 | # The class to use for running task instances in a subprocess 148 | task_runner = StandardTaskRunner 149 | 150 | # If set, tasks without a ``run_as_user`` argument will be run with this user 151 | # Can be used to de-elevate a sudo user running Airflow when executing tasks 152 | default_impersonation = airflow 153 | 154 | # What security module to use (for example kerberos) 155 | security = 156 | 157 | # If set to False enables some unsecure features like Charts and Ad Hoc Queries. 158 | # In 2.0 will default to True. 159 | secure_mode = False 160 | 161 | # Turn unit test mode on (overwrites many configuration options with test 162 | # values at runtime) 163 | unit_test_mode = False 164 | 165 | # Whether to enable pickling for xcom (note that this is insecure and allows for 166 | # RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False). 167 | enable_xcom_pickling = True 168 | 169 | # When a task is killed forcefully, this is the amount of time in seconds that 170 | # it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED 171 | killed_task_cleanup_time = 60 172 | 173 | # Whether to override params with dag_run.conf. If you pass some key-value pairs 174 | # through ``airflow dags backfill -c`` or 175 | # ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. 176 | dag_run_conf_overrides_params = False 177 | 178 | # Worker initialisation check to validate Metadata Database connection 179 | worker_precheck = False 180 | 181 | # When discovering DAGs, ignore any files that don't contain the strings ``DAG`` and ``airflow``. 182 | dag_discovery_safe_mode = True 183 | 184 | # The number of retries each task is going to have by default. Can be overridden at dag or task level. 185 | default_task_retries = 0 186 | 187 | # Whether to serialises DAGs and persist them in DB. 188 | # If set to True, Webserver reads from DB instead of parsing DAG files 189 | # More details: https://airflow.apache.org/docs/stable/dag-serialization.html 190 | store_serialized_dags = False 191 | 192 | # Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. 193 | min_serialized_dag_update_interval = 30 194 | 195 | # On each dagrun check against defined SLAs 196 | check_slas = True 197 | 198 | [cli] 199 | # In what way should the cli access the API. The LocalClient will use the 200 | # database directly, while the json_client will use the api running on the 201 | # webserver 202 | api_client = airflow.api.client.local_client 203 | 204 | # If you set web_server_url_prefix, do NOT forget to append it here, ex: 205 | # ``endpoint_url = http://localhost:8080/myroot`` 206 | # So api will look like: ``http://localhost:8080/myroot/api/experimental/...`` 207 | endpoint_url = http://localhost:8080 208 | 209 | [debug] 210 | # Used only with DebugExecutor. If set to True DAG will fail with first 211 | # failed task. Helpful for debugging purposes. 212 | fail_fast = False 213 | 214 | [api] 215 | # How to authenticate users of the API 216 | auth_backend = airflow.api.auth.backend.default 217 | 218 | [lineage] 219 | # what lineage backend to use 220 | backend = 221 | 222 | [atlas] 223 | sasl_enabled = False 224 | host = 225 | port = 21000 226 | username = 227 | password = 228 | 229 | [operators] 230 | # The default owner assigned to each new operator, unless 231 | # provided explicitly or passed via ``default_args`` 232 | default_owner = airflow 233 | default_cpus = 1 234 | default_ram = 512 235 | default_disk = 512 236 | default_gpus = 0 237 | 238 | [hive] 239 | # Default mapreduce queue for HiveOperator tasks 240 | default_hive_mapred_queue = 241 | 242 | [webserver] 243 | # The base url of your website as airflow cannot guess what domain or 244 | # cname you are using. This is used in automated emails that 245 | # airflow sends to point links to the right web server 246 | base_url = http://localhost:8080 247 | 248 | # The ip specified when starting the web server 249 | web_server_host = 0.0.0.0 250 | 251 | # The port on which to run the web server 252 | web_server_port = 8080 253 | 254 | # Paths to the SSL certificate and key for the web server. When both are 255 | # provided SSL will be enabled. This does not change the web server port. 256 | web_server_ssl_cert = 257 | 258 | # Paths to the SSL certificate and key for the web server. When both are 259 | # provided SSL will be enabled. This does not change the web server port. 260 | web_server_ssl_key = 261 | 262 | # Number of seconds the webserver waits before killing gunicorn master that doesn't respond 263 | web_server_master_timeout = 120 264 | 265 | # Number of seconds the gunicorn webserver waits before timing out on a worker 266 | web_server_worker_timeout = 120 267 | 268 | # Number of workers to refresh at a time. When set to 0, worker refresh is 269 | # disabled. When nonzero, airflow periodically refreshes webserver workers by 270 | # bringing up new ones and killing old ones. 271 | worker_refresh_batch_size = 1 272 | 273 | # Number of seconds to wait before refreshing a batch of workers. 274 | worker_refresh_interval = 30 275 | 276 | # Secret key used to run your flask app 277 | # It should be as random as possible 278 | secret_key = temporary_key 279 | 280 | # Number of workers to run the Gunicorn web server 281 | workers = 4 282 | 283 | # The worker class gunicorn should use. Choices include 284 | # sync (default), eventlet, gevent 285 | worker_class = sync 286 | 287 | # Log files for the gunicorn webserver. '-' means log to stderr. 288 | access_logfile = - 289 | 290 | # Log files for the gunicorn webserver. '-' means log to stderr. 291 | error_logfile = - 292 | 293 | # Expose the configuration file in the web server 294 | expose_config = True 295 | 296 | # Expose hostname in the web server 297 | expose_hostname = True 298 | 299 | # Expose stacktrace in the web server 300 | expose_stacktrace = True 301 | 302 | # Set to true to turn on authentication: 303 | # https://airflow.apache.org/security.html#web-authentication 304 | authenticate = False 305 | 306 | # Filter the list of dags by owner name (requires authentication to be enabled) 307 | filter_by_owner = False 308 | 309 | # Filtering mode. Choices include user (default) and ldapgroup. 310 | # Ldap group filtering requires using the ldap backend 311 | # 312 | # Note that the ldap server needs the "memberOf" overlay to be set up 313 | # in order to user the ldapgroup mode. 314 | owner_mode = user 315 | 316 | # Default DAG view. Valid values are: 317 | # tree, graph, duration, gantt, landing_times 318 | dag_default_view = tree 319 | 320 | # "Default DAG orientation. Valid values are:" 321 | # LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top) 322 | dag_orientation = LR 323 | 324 | # Puts the webserver in demonstration mode; blurs the names of Operators for 325 | # privacy. 326 | demo_mode = False 327 | 328 | # The amount of time (in secs) webserver will wait for initial handshake 329 | # while fetching logs from other worker machine 330 | log_fetch_timeout_sec = 5 331 | 332 | # Time interval (in secs) to wait before next log fetching. 333 | log_fetch_delay_sec = 2 334 | 335 | # Distance away from page bottom to enable auto tailing. 336 | log_auto_tailing_offset = 30 337 | 338 | # Animation speed for auto tailing log display. 339 | log_animation_speed = 1000 340 | 341 | # By default, the webserver shows paused DAGs. Flip this to hide paused 342 | # DAGs by default 343 | hide_paused_dags_by_default = False 344 | 345 | # Consistent page size across all listing views in the UI 346 | page_size = 100 347 | 348 | # Use FAB-based webserver with RBAC feature 349 | rbac = False 350 | 351 | # Define the color of navigation bar 352 | navbar_color = #007A87 353 | 354 | # Default dagrun to show in UI 355 | default_dag_run_display_number = 25 356 | 357 | # Enable werkzeug ``ProxyFix`` middleware for reverse proxy 358 | enable_proxy_fix = False 359 | 360 | # Number of values to trust for ``X-Forwarded-For``. 361 | # More info: https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/ 362 | proxy_fix_x_for = 1 363 | 364 | # Number of values to trust for ``X-Forwarded-Proto`` 365 | proxy_fix_x_proto = 1 366 | 367 | # Number of values to trust for ``X-Forwarded-Host`` 368 | proxy_fix_x_host = 1 369 | 370 | # Number of values to trust for ``X-Forwarded-Port`` 371 | proxy_fix_x_port = 1 372 | 373 | # Number of values to trust for ``X-Forwarded-Prefix`` 374 | proxy_fix_x_prefix = 1 375 | 376 | # Set secure flag on session cookie 377 | cookie_secure = False 378 | 379 | # Set samesite policy on session cookie 380 | cookie_samesite = 381 | 382 | # Default setting for wrap toggle on DAG code and TI log views. 383 | default_wrap = False 384 | 385 | # Allow the UI to be rendered in a frame 386 | x_frame_enabled = True 387 | 388 | # Send anonymous user activity to your analytics tool 389 | # choose from google_analytics, segment, or metarouter 390 | # analytics_tool = 391 | 392 | # Unique ID of your account in the analytics tool 393 | # analytics_id = 394 | 395 | # Update FAB permissions and sync security manager roles 396 | # on webserver startup 397 | update_fab_perms = True 398 | 399 | # Minutes of non-activity before logged out from UI 400 | # 0 means never get forcibly logged out 401 | force_log_out_after = 0 402 | 403 | # The UI cookie lifetime in days 404 | session_lifetime_days = 30 405 | 406 | [email] 407 | email_backend = airflow.utils.email.send_email_smtp 408 | 409 | [smtp] 410 | 411 | # If you want airflow to send emails on retries, failure, and you want to use 412 | # the airflow.utils.email.send_email_smtp function, you have to configure an 413 | # smtp server here 414 | smtp_host = localhost 415 | smtp_starttls = True 416 | smtp_ssl = False 417 | # Example: smtp_user = airflow 418 | # smtp_user = 419 | # Example: smtp_password = airflow 420 | # smtp_password = 421 | smtp_port = 25 422 | smtp_mail_from = airflow@example.com 423 | 424 | [sentry] 425 | 426 | # Sentry (https://docs.sentry.io) integration 427 | sentry_dsn = 428 | 429 | [celery] 430 | 431 | # This section only applies if you are using the CeleryExecutor in 432 | # ``[core]`` section above 433 | # The app name that will be used by celery 434 | celery_app_name = airflow.executors.celery_executor 435 | 436 | # The concurrency that will be used when starting workers with the 437 | # ``airflow celery worker`` command. This defines the number of task instances that 438 | # a worker will take, so size up your workers based on the resources on 439 | # your worker box and the nature of your tasks 440 | worker_concurrency = 16 441 | 442 | # The maximum and minimum concurrency that will be used when starting workers with the 443 | # ``airflow celery worker`` command (always keep minimum processes, but grow 444 | # to maximum if necessary). Note the value should be max_concurrency,min_concurrency 445 | # Pick these numbers based on resources on worker box and the nature of the task. 446 | # If autoscale option is available, worker_concurrency will be ignored. 447 | # http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale 448 | # Example: worker_autoscale = 16,12 449 | worker_autoscale = 16,12 450 | 451 | # When you start an airflow worker, airflow starts a tiny web server 452 | # subprocess to serve the workers local log files to the airflow main 453 | # web server, who then builds pages and sends them to users. This defines 454 | # the port on which the logs are served. It needs to be unused, and open 455 | # visible from the main web server to connect into the workers. 456 | worker_log_server_port = 8793 457 | 458 | # The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally 459 | # a sqlalchemy database. Refer to the Celery documentation for more 460 | # information. 461 | # http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings 462 | broker_url = redis://redis:6379/1 463 | 464 | # The Celery result_backend. When a job finishes, it needs to update the 465 | # metadata of the job. Therefore it will post a message on a message bus, 466 | # or insert it into a database (depending of the backend) 467 | # This status is used by the scheduler to update the state of the task 468 | # The use of a database is highly recommended 469 | # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings 470 | result_backend = db+postgresql://airflow:airflow@postgres/airflow 471 | 472 | # Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start 473 | # it ``airflow flower``. This defines the IP that Celery Flower runs on 474 | flower_host = 0.0.0.0 475 | 476 | # The root URL for Flower 477 | # Example: flower_url_prefix = /flower 478 | flower_url_prefix = 479 | 480 | # This defines the port that Celery Flower runs on 481 | flower_port = 5555 482 | 483 | # Securing Flower with Basic Authentication 484 | # Accepts user:password pairs separated by a comma 485 | # Example: flower_basic_auth = user1:password1,user2:password2 486 | flower_basic_auth = 487 | 488 | # Default queue that tasks get assigned to and that worker listen on. 489 | default_queue = default 490 | 491 | # How many processes CeleryExecutor uses to sync task state. 492 | # 0 means to use max(1, number of cores - 1) processes. 493 | sync_parallelism = 0 494 | 495 | # Import path for celery configuration options 496 | celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG 497 | 498 | # In case of using SSL 499 | ssl_active = False 500 | ssl_key = 501 | ssl_cert = 502 | ssl_cacert = 503 | 504 | # Celery Pool implementation. 505 | # Choices include: prefork (default), eventlet, gevent or solo. 506 | # See: 507 | # https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency 508 | # https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html 509 | pool = prefork 510 | 511 | # The number of seconds to wait before timing out ``send_task_to_executor`` or 512 | # ``fetch_celery_task_state`` operations. 513 | operation_timeout = 2 514 | 515 | [celery_broker_transport_options] 516 | 517 | # This section is for specifying options which can be passed to the 518 | # underlying celery broker transport. See: 519 | # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options 520 | # The visibility timeout defines the number of seconds to wait for the worker 521 | # to acknowledge the task before the message is redelivered to another worker. 522 | # Make sure to increase the visibility timeout to match the time of the longest 523 | # ETA you're planning to use. 524 | # visibility_timeout is only supported for Redis and SQS celery brokers. 525 | # See: 526 | # http://docs.celeryproject.org/en/master/userguide/configuration.html#std:setting-broker_transport_options 527 | # Example: visibility_timeout = 21600 528 | # visibility_timeout = 529 | 530 | [dask] 531 | 532 | # This section only applies if you are using the DaskExecutor in 533 | # [core] section above 534 | # The IP address and port of the Dask cluster's scheduler. 535 | cluster_address = 127.0.0.1:8786 536 | 537 | # TLS/ SSL settings to access a secured Dask scheduler. 538 | tls_ca = 539 | tls_cert = 540 | tls_key = 541 | 542 | [scheduler] 543 | # Task instances listen for external kill signal (when you clear tasks 544 | # from the CLI or the UI), this defines the frequency at which they should 545 | # listen (in seconds). 546 | job_heartbeat_sec = 5 547 | 548 | # The scheduler constantly tries to trigger new tasks (look at the 549 | # scheduler section in the docs for more information). This defines 550 | # how often the scheduler should run (in seconds). 551 | scheduler_heartbeat_sec = 5 552 | 553 | # After how much time should the scheduler terminate in seconds 554 | # -1 indicates to run continuously (see also num_runs) 555 | run_duration = -1 556 | 557 | # The number of times to try to schedule each DAG file 558 | # -1 indicates unlimited number 559 | num_runs = -1 560 | 561 | # The number of seconds to wait between consecutive DAG file processing 562 | processor_poll_interval = 1 563 | 564 | # after how much time (seconds) a new DAGs should be picked up from the filesystem 565 | min_file_process_interval = 0 566 | 567 | # How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. 568 | dag_dir_list_interval = 300 569 | 570 | # How often should stats be printed to the logs. Setting to 0 will disable printing stats 571 | print_stats_interval = 30 572 | 573 | # If the last scheduler heartbeat happened more than scheduler_health_check_threshold 574 | # ago (in seconds), scheduler is considered unhealthy. 575 | # This is used by the health check in the "/health" endpoint 576 | scheduler_health_check_threshold = 30 577 | child_process_log_directory = /usr/local/airflow/logs/scheduler 578 | 579 | # Local task jobs periodically heartbeat to the DB. If the job has 580 | # not heartbeat in this many seconds, the scheduler will mark the 581 | # associated task instance as failed and will re-schedule the task. 582 | scheduler_zombie_task_threshold = 300 583 | 584 | # Turn off scheduler catchup by setting this to False. 585 | # Default behavior is unchanged and 586 | # Command Line Backfills still work, but the scheduler 587 | # will not do scheduler catchup if this is False, 588 | # however it can be set on a per DAG basis in the 589 | # DAG definition (catchup) 590 | catchup_by_default = True 591 | 592 | # This changes the batch size of queries in the scheduling main loop. 593 | # If this is too high, SQL query performance may be impacted by one 594 | # or more of the following: 595 | # - reversion to full table scan 596 | # - complexity of query predicate 597 | # - excessive locking 598 | # Additionally, you may hit the maximum allowable query length for your db. 599 | # Set this to 0 for no limit (not advised) 600 | max_tis_per_query = 512 601 | 602 | # Statsd (https://github.com/etsy/statsd) integration settings 603 | statsd_on = False 604 | statsd_host = localhost 605 | statsd_port = 8125 606 | statsd_prefix = airflow 607 | 608 | # If you want to avoid send all the available metrics to StatsD, 609 | # you can configure an allow list of prefixes to send only the metrics that 610 | # start with the elements of the list (e.g: scheduler,executor,dagrun) 611 | statsd_allow_list = 612 | 613 | # The scheduler can run multiple threads in parallel to schedule dags. 614 | # This defines how many threads will run. 615 | max_threads = 2 616 | authenticate = False 617 | 618 | # Turn off scheduler use of cron intervals by setting this to False. 619 | # DAGs submitted manually in the web UI or with trigger_dag will still run. 620 | use_job_schedule = True 621 | 622 | # Allow externally triggered DagRuns for Execution Dates in the future 623 | # Only has effect if schedule_interval is set to None in DAG 624 | allow_trigger_in_future = False 625 | 626 | [ldap] 627 | # set this to ldaps://: 628 | uri = 629 | user_filter = objectClass=* 630 | user_name_attr = uid 631 | group_member_attr = memberOf 632 | superuser_filter = 633 | data_profiler_filter = 634 | bind_user = cn=Manager,dc=example,dc=com 635 | bind_password = insecure 636 | basedn = dc=example,dc=com 637 | cacert = /etc/ca/ldap_ca.crt 638 | search_scope = LEVEL 639 | 640 | # This setting allows the use of LDAP servers that either return a 641 | # broken schema, or do not return a schema. 642 | ignore_malformed_schema = False 643 | 644 | [mesos] 645 | # Mesos master address which MesosExecutor will connect to. 646 | master = localhost:5050 647 | 648 | # The framework name which Airflow scheduler will register itself as on mesos 649 | framework_name = Airflow 650 | 651 | # Number of cpu cores required for running one task instance using 652 | # 'airflow run --local -p ' 653 | # command on a mesos slave 654 | task_cpu = 1 655 | 656 | # Memory in MB required for running one task instance using 657 | # 'airflow run --local -p ' 658 | # command on a mesos slave 659 | task_memory = 256 660 | 661 | # Enable framework checkpointing for mesos 662 | # See http://mesos.apache.org/documentation/latest/slave-recovery/ 663 | checkpoint = False 664 | 665 | # Failover timeout in milliseconds. 666 | # When checkpointing is enabled and this option is set, Mesos waits 667 | # until the configured timeout for 668 | # the MesosExecutor framework to re-register after a failover. Mesos 669 | # shuts down running tasks if the 670 | # MesosExecutor framework fails to re-register within this timeframe. 671 | # Example: failover_timeout = 604800 672 | # failover_timeout = 673 | 674 | # Enable framework authentication for mesos 675 | # See http://mesos.apache.org/documentation/latest/configuration/ 676 | authenticate = False 677 | 678 | # Mesos credentials, if authentication is enabled 679 | # Example: default_principal = admin 680 | # default_principal = 681 | # Example: default_secret = admin 682 | # default_secret = 683 | 684 | # Optional Docker Image to run on slave before running the command 685 | # This image should be accessible from mesos slave i.e mesos slave 686 | # should be able to pull this docker image before executing the command. 687 | # Example: docker_image_slave = puckel/docker-airflow 688 | # docker_image_slave = 689 | 690 | [kerberos] 691 | ccache = /tmp/airflow_krb5_ccache 692 | 693 | # gets augmented with fqdn 694 | principal = airflow 695 | reinit_frequency = 3600 696 | kinit_path = kinit 697 | keytab = airflow.keytab 698 | 699 | [github_enterprise] 700 | api_rev = v3 701 | 702 | [admin] 703 | # UI to hide sensitive variable fields when set to True 704 | hide_sensitive_variable_fields = True 705 | 706 | [elasticsearch] 707 | # Elasticsearch host 708 | host = 709 | 710 | # Format of the log_id, which is used to query for a given tasks logs 711 | log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}} 712 | 713 | # Used to mark the end of a log stream for a task 714 | end_of_log_mark = end_of_log 715 | 716 | # Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id 717 | # Code will construct log_id using the log_id template from the argument above. 718 | # NOTE: The code will prefix the https:// automatically, don't include that here. 719 | frontend = 720 | 721 | # Write the task logs to the stdout of the worker, rather than the default files 722 | write_stdout = False 723 | 724 | # Instead of the default log formatter, write the log lines as JSON 725 | json_format = False 726 | 727 | # Log fields to also attach to the json output, if enabled 728 | json_fields = asctime, filename, lineno, levelname, message 729 | 730 | [elasticsearch_configs] 731 | use_ssl = False 732 | verify_certs = True 733 | 734 | [kubernetes] 735 | # The repository, tag and imagePullPolicy of the Kubernetes Image for the Worker to Run 736 | worker_container_repository = 737 | worker_container_tag = 738 | worker_container_image_pull_policy = IfNotPresent 739 | 740 | # If True (default), worker pods will be deleted upon termination 741 | delete_worker_pods = True 742 | 743 | # Number of Kubernetes Worker Pod creation calls per scheduler loop 744 | worker_pods_creation_batch_size = 1 745 | 746 | # The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` 747 | namespace = default 748 | 749 | # The name of the Kubernetes ConfigMap containing the Airflow Configuration (this file) 750 | # Example: airflow_configmap = airflow-configmap 751 | airflow_configmap = 752 | 753 | # The name of the Kubernetes ConfigMap containing ``airflow_local_settings.py`` file. 754 | # 755 | # For example: 756 | # 757 | # ``airflow_local_settings_configmap = "airflow-configmap"`` if you have the following ConfigMap. 758 | # 759 | # ``airflow-configmap.yaml``: 760 | # 761 | # .. code-block:: yaml 762 | # 763 | # --- 764 | # apiVersion: v1 765 | # kind: ConfigMap 766 | # metadata: 767 | # name: airflow-configmap 768 | # data: 769 | # airflow_local_settings.py: | 770 | # def pod_mutation_hook(pod): 771 | # ... 772 | # airflow.cfg: | 773 | # ... 774 | # Example: airflow_local_settings_configmap = airflow-configmap 775 | airflow_local_settings_configmap = 776 | 777 | # For docker image already contains DAGs, this is set to ``True``, and the worker will 778 | # search for dags in dags_folder, 779 | # otherwise use git sync or dags volume claim to mount DAGs 780 | dags_in_image = False 781 | 782 | # For either git sync or volume mounted DAGs, the worker will look in this subpath for DAGs 783 | dags_volume_subpath = 784 | 785 | # For DAGs mounted via a volume claim (mutually exclusive with git-sync and host path) 786 | dags_volume_claim = 787 | 788 | # For volume mounted logs, the worker will look in this subpath for logs 789 | logs_volume_subpath = 790 | 791 | # A shared volume claim for the logs 792 | logs_volume_claim = 793 | 794 | # For DAGs mounted via a hostPath volume (mutually exclusive with volume claim and git-sync) 795 | # Useful in local environment, discouraged in production 796 | dags_volume_host = 797 | 798 | # A hostPath volume for the logs 799 | # Useful in local environment, discouraged in production 800 | logs_volume_host = 801 | 802 | # A list of configMapsRefs to envFrom. If more than one configMap is 803 | # specified, provide a comma separated list: configmap_a,configmap_b 804 | env_from_configmap_ref = 805 | 806 | # A list of secretRefs to envFrom. If more than one secret is 807 | # specified, provide a comma separated list: secret_a,secret_b 808 | env_from_secret_ref = 809 | 810 | # Git credentials and repository for DAGs mounted via Git (mutually exclusive with volume claim) 811 | git_repo = 812 | git_branch = 813 | git_subpath = 814 | 815 | # The specific rev or hash the git_sync init container will checkout 816 | # This becomes GIT_SYNC_REV environment variable in the git_sync init container for worker pods 817 | git_sync_rev = 818 | 819 | # Use git_user and git_password for user authentication or git_ssh_key_secret_name 820 | # and git_ssh_key_secret_key for SSH authentication 821 | git_user = 822 | git_password = 823 | git_sync_root = /git 824 | git_sync_dest = repo 825 | 826 | # Mount point of the volume if git-sync is being used. 827 | # i.e. /usr/local/airflow/dags 828 | git_dags_folder_mount_point = 829 | 830 | # To get Git-sync SSH authentication set up follow this format 831 | # 832 | # ``airflow-secrets.yaml``: 833 | # 834 | # .. code-block:: yaml 835 | # 836 | # --- 837 | # apiVersion: v1 838 | # kind: Secret 839 | # metadata: 840 | # name: airflow-secrets 841 | # data: 842 | # # key needs to be gitSshKey 843 | # gitSshKey: 844 | # Example: git_ssh_key_secret_name = airflow-secrets 845 | git_ssh_key_secret_name = 846 | 847 | # To get Git-sync SSH authentication set up follow this format 848 | # 849 | # ``airflow-configmap.yaml``: 850 | # 851 | # .. code-block:: yaml 852 | # 853 | # --- 854 | # apiVersion: v1 855 | # kind: ConfigMap 856 | # metadata: 857 | # name: airflow-configmap 858 | # data: 859 | # known_hosts: | 860 | # github.com ssh-rsa <...> 861 | # airflow.cfg: | 862 | # ... 863 | # Example: git_ssh_known_hosts_configmap_name = airflow-configmap 864 | git_ssh_known_hosts_configmap_name = 865 | 866 | # To give the git_sync init container credentials via a secret, create a secret 867 | # with two fields: GIT_SYNC_USERNAME and GIT_SYNC_PASSWORD (example below) and 868 | # add ``git_sync_credentials_secret = `` to your airflow config under the 869 | # ``kubernetes`` section 870 | # 871 | # Secret Example: 872 | # 873 | # .. code-block:: yaml 874 | # 875 | # --- 876 | # apiVersion: v1 877 | # kind: Secret 878 | # metadata: 879 | # name: git-credentials 880 | # data: 881 | # GIT_SYNC_USERNAME: 882 | # GIT_SYNC_PASSWORD: 883 | git_sync_credentials_secret = 884 | 885 | # For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync 886 | git_sync_container_repository = k8s.gcr.io/git-sync 887 | git_sync_container_tag = v3.1.1 888 | git_sync_init_container_name = git-sync-clone 889 | git_sync_run_as_user = 65533 890 | 891 | # The name of the Kubernetes service account to be associated with airflow workers, if any. 892 | # Service accounts are required for workers that require access to secrets or cluster resources. 893 | # See the Kubernetes RBAC documentation for more: 894 | # https://kubernetes.io/docs/admin/authorization/rbac/ 895 | worker_service_account_name = 896 | 897 | # Any image pull secrets to be given to worker pods, If more than one secret is 898 | # required, provide a comma separated list: secret_a,secret_b 899 | image_pull_secrets = 900 | 901 | # GCP Service Account Keys to be provided to tasks run on Kubernetes Executors 902 | # Should be supplied in the format: key-name-1:key-path-1,key-name-2:key-path-2 903 | gcp_service_account_keys = 904 | 905 | # Use the service account kubernetes gives to pods to connect to kubernetes cluster. 906 | # It's intended for clients that expect to be running inside a pod running on kubernetes. 907 | # It will raise an exception if called from a process not running in a kubernetes environment. 908 | in_cluster = True 909 | 910 | # When running with in_cluster=False change the default cluster_context or config_file 911 | # options to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has. 912 | # cluster_context = 913 | # config_file = 914 | 915 | # Affinity configuration as a single line formatted JSON object. 916 | # See the affinity model for top-level key names (e.g. ``nodeAffinity``, etc.): 917 | # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#affinity-v1-core 918 | affinity = 919 | 920 | # A list of toleration objects as a single line formatted JSON array 921 | # See: 922 | # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#toleration-v1-core 923 | tolerations = 924 | 925 | # Keyword parameters to pass while calling a kubernetes client core_v1_api methods 926 | # from Kubernetes Executor provided as a single line formatted JSON dictionary string. 927 | # List of supported params are similar for all core_v1_apis, hence a single config 928 | # variable for all apis. 929 | # See: 930 | # https://raw.githubusercontent.com/kubernetes-client/python/master/kubernetes/client/apis/core_v1_api.py 931 | # Note that if no _request_timeout is specified, the kubernetes client will wait indefinitely 932 | # for kubernetes api responses, which will cause the scheduler to hang. 933 | # The timeout is specified as [connect timeout, read timeout] 934 | kube_client_request_args = {{"_request_timeout" : [60,60] }} 935 | 936 | # Specifies the uid to run the first process of the worker pods containers as 937 | run_as_user = 938 | 939 | # Specifies a gid to associate with all containers in the worker pods 940 | # if using a git_ssh_key_secret_name use an fs_group 941 | # that allows for the key to be read, e.g. 65533 942 | fs_group = 943 | 944 | [kubernetes_node_selectors] 945 | 946 | # The Key-value pairs to be given to worker pods. 947 | # The worker pods will be scheduled to the nodes of the specified key-value pairs. 948 | # Should be supplied in the format: key = value 949 | 950 | [kubernetes_annotations] 951 | 952 | # The Key-value annotations pairs to be given to worker pods. 953 | # Should be supplied in the format: key = value 954 | 955 | [kubernetes_environment_variables] 956 | 957 | # The scheduler sets the following environment variables into your workers. You may define as 958 | # many environment variables as needed and the kubernetes launcher will set them in the launched workers. 959 | # Environment variables in this section are defined as follows 960 | # `` = `` 961 | # 962 | # For example if you wanted to set an environment variable with value `prod` and key 963 | # ``ENVIRONMENT`` you would follow the following format: 964 | # ENVIRONMENT = prod 965 | # 966 | # Additionally you may override worker airflow settings with the ``AIRFLOW__
__`` 967 | # formatting as supported by airflow normally. 968 | 969 | [kubernetes_secrets] 970 | 971 | # The scheduler mounts the following secrets into your workers as they are launched by the 972 | # scheduler. You may define as many secrets as needed and the kubernetes launcher will parse the 973 | # defined secrets and mount them as secret environment variables in the launched workers. 974 | # Secrets in this section are defined as follows 975 | # `` = =`` 976 | # 977 | # For example if you wanted to mount a kubernetes secret key named ``postgres_password`` from the 978 | # kubernetes secret object ``airflow-secret`` as the environment variable ``POSTGRES_PASSWORD`` into 979 | # your workers you would follow the following format: 980 | # ``POSTGRES_PASSWORD = airflow-secret=postgres_credentials`` 981 | # 982 | # Additionally you may override worker airflow settings with the ``AIRFLOW__
__`` 983 | # formatting as supported by airflow normally. 984 | 985 | [kubernetes_labels] 986 | 987 | # The Key-value pairs to be given to worker pods. 988 | # The worker pods will be given these static labels, as well as some additional dynamic labels 989 | # to identify the task. 990 | # Should be supplied in the format: ``key = value`` 991 | -------------------------------------------------------------------------------- /dags/scripts/spark/random_text_classification.py: -------------------------------------------------------------------------------- 1 | # pyspark 2 | import argparse 3 | 4 | from pyspark.sql import SparkSession 5 | from pyspark.ml.feature import Tokenizer, StopWordsRemover 6 | from pyspark.sql.functions import array_contains 7 | 8 | 9 | def random_text_classifier(input_loc, output_loc): 10 | """ 11 | This is a dummy function to show how to use spark, It is supposed to mock 12 | the following steps 13 | 1. clean input data 14 | 2. use a pre-trained model to make prediction 15 | 3. write predictions to a HDFS output 16 | 17 | Since this is meant as an example, we are going to skip building a model, 18 | instead we are naively going to mark reviews having the text "good" as positive and 19 | the rest as negative 20 | """ 21 | 22 | # read input 23 | df_raw = spark.read.option("header", True).csv(input_loc) 24 | # perform text cleaning 25 | 26 | # Tokenize text 27 | tokenizer = Tokenizer(inputCol="review_str", outputCol="review_token") 28 | df_tokens = tokenizer.transform(df_raw).select("cid", "review_token") 29 | 30 | # Remove stop words 31 | remover = StopWordsRemover(inputCol="review_token", outputCol="review_clean") 32 | df_clean = remover.transform(df_tokens).select("cid", "review_clean") 33 | 34 | # function to check presence of good 35 | df_out = df_clean.select( 36 | "cid", array_contains(df_clean.review_clean, "good").alias("positive_review") 37 | ) 38 | # parquet is a popular column storage format, we use it here 39 | df_out.write.mode("overwrite").parquet(output_loc) 40 | 41 | 42 | if __name__ == "__main__": 43 | parser = argparse.ArgumentParser() 44 | parser.add_argument("--input", type=str, help="HDFS input", default="/movie") 45 | parser.add_argument("--output", type=str, help="HDFS output", default="/output") 46 | args = parser.parse_args() 47 | spark = SparkSession.builder.appName("Random Text Classifier").getOrCreate() 48 | random_text_classifier(input_loc=args.input, output_loc=args.output) 49 | -------------------------------------------------------------------------------- /dags/spark_submit_airflow.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | from airflow import DAG 4 | from airflow.operators.dummy_operator import DummyOperator 5 | from airflow.hooks.S3_hook import S3Hook 6 | from airflow.operators import PythonOperator 7 | from airflow.contrib.operators.emr_create_job_flow_operator import ( 8 | EmrCreateJobFlowOperator, 9 | ) 10 | from airflow.contrib.operators.emr_add_steps_operator import EmrAddStepsOperator 11 | from airflow.contrib.sensors.emr_step_sensor import EmrStepSensor 12 | from airflow.contrib.operators.emr_terminate_job_flow_operator import ( 13 | EmrTerminateJobFlowOperator, 14 | ) 15 | 16 | # Configurations 17 | BUCKET_NAME = "" # replace this with your bucket name 18 | local_data = "./dags/data/movie_review.csv" 19 | s3_data = "data/movie_review.csv" 20 | local_script = "./dags/scripts/spark/random_text_classification.py" 21 | s3_script = "scripts/random_text_classification.py" 22 | s3_clean = "clean_data/" 23 | SPARK_STEPS = [ 24 | { 25 | "Name": "Move raw data from S3 to HDFS", 26 | "ActionOnFailure": "CANCEL_AND_WAIT", 27 | "HadoopJarStep": { 28 | "Jar": "command-runner.jar", 29 | "Args": [ 30 | "s3-dist-cp", 31 | "--src=s3://{{ params.BUCKET_NAME }}/data", 32 | "--dest=/movie", 33 | ], 34 | }, 35 | }, 36 | { 37 | "Name": "Classify movie reviews", 38 | "ActionOnFailure": "CANCEL_AND_WAIT", 39 | "HadoopJarStep": { 40 | "Jar": "command-runner.jar", 41 | "Args": [ 42 | "spark-submit", 43 | "--deploy-mode", 44 | "client", 45 | "s3://{{ params.BUCKET_NAME }}/{{ params.s3_script }}", 46 | ], 47 | }, 48 | }, 49 | { 50 | "Name": "Move clean data from HDFS to S3", 51 | "ActionOnFailure": "CANCEL_AND_WAIT", 52 | "HadoopJarStep": { 53 | "Jar": "command-runner.jar", 54 | "Args": [ 55 | "s3-dist-cp", 56 | "--src=/output", 57 | "--dest=s3://{{ params.BUCKET_NAME }}/{{ params.s3_clean }}", 58 | ], 59 | }, 60 | }, 61 | ] 62 | 63 | JOB_FLOW_OVERRIDES = { 64 | "Name": "Movie review classifier", 65 | "ReleaseLabel": "emr-5.29.0", 66 | "Applications": [{"Name": "Hadoop"}, {"Name": "Spark"}], 67 | "Configurations": [ 68 | { 69 | "Classification": "spark-env", 70 | "Configurations": [ 71 | { 72 | "Classification": "export", 73 | "Properties": {"PYSPARK_PYTHON": "/usr/bin/python3"}, 74 | } 75 | ], 76 | } 77 | ], 78 | "Instances": { 79 | "InstanceGroups": [ 80 | { 81 | "Name": "Master node", 82 | "Market": "SPOT", 83 | "InstanceRole": "MASTER", 84 | "InstanceType": "m4.xlarge", 85 | "InstanceCount": 1, 86 | }, 87 | { 88 | "Name": "Core - 2", 89 | "Market": "SPOT", 90 | "InstanceRole": "CORE", 91 | "InstanceType": "m4.xlarge", 92 | "InstanceCount": 2, 93 | }, 94 | ], 95 | "KeepJobFlowAliveWhenNoSteps": True, 96 | "TerminationProtected": False, 97 | }, 98 | "JobFlowRole": "EMR_EC2_DefaultRole", 99 | "ServiceRole": "EMR_DefaultRole", 100 | } 101 | 102 | # helper function 103 | def _local_to_s3(filename, key, bucket_name=BUCKET_NAME): 104 | s3 = S3Hook() 105 | s3.load_file(filename=filename, bucket_name=bucket_name, replace=True, key=key) 106 | 107 | 108 | default_args = { 109 | "owner": "airflow", 110 | "depends_on_past": True, 111 | "wait_for_downstream": True, 112 | "start_date": datetime(2020, 10, 17), 113 | "email": ["airflow@airflow.com"], 114 | "email_on_failure": False, 115 | "email_on_retry": False, 116 | "retries": 1, 117 | "retry_delay": timedelta(minutes=5), 118 | } 119 | 120 | dag = DAG( 121 | "spark_submit_airflow", 122 | default_args=default_args, 123 | schedule_interval="0 10 * * *", 124 | max_active_runs=1, 125 | ) 126 | 127 | start_data_pipeline = DummyOperator(task_id="start_data_pipeline", dag=dag) 128 | 129 | data_to_s3 = PythonOperator( 130 | dag=dag, 131 | task_id="data_to_s3", 132 | python_callable=_local_to_s3, 133 | op_kwargs={"filename": local_data, "key": s3_data,}, 134 | ) 135 | 136 | script_to_s3 = PythonOperator( 137 | dag=dag, 138 | task_id="script_to_s3", 139 | python_callable=_local_to_s3, 140 | op_kwargs={"filename": local_script, "key": s3_script,}, 141 | ) 142 | 143 | # Create an EMR cluster 144 | create_emr_cluster = EmrCreateJobFlowOperator( 145 | task_id="create_emr_cluster", 146 | job_flow_overrides=JOB_FLOW_OVERRIDES, 147 | aws_conn_id="aws_default", 148 | emr_conn_id="emr_default", 149 | dag=dag, 150 | ) 151 | 152 | # Add your steps to the EMR cluster 153 | step_adder = EmrAddStepsOperator( 154 | task_id="add_steps", 155 | job_flow_id="{{ task_instance.xcom_pull(task_ids='create_emr_cluster', key='return_value') }}", 156 | aws_conn_id="aws_default", 157 | steps=SPARK_STEPS, 158 | params={ 159 | "BUCKET_NAME": BUCKET_NAME, 160 | "s3_data": s3_data, 161 | "s3_script": s3_script, 162 | "s3_clean": s3_clean, 163 | }, 164 | dag=dag, 165 | ) 166 | 167 | last_step = len(SPARK_STEPS) - 1 168 | # wait for the steps to complete 169 | step_checker = EmrStepSensor( 170 | task_id="watch_step", 171 | job_flow_id="{{ task_instance.xcom_pull('create_emr_cluster', key='return_value') }}", 172 | step_id="{{ task_instance.xcom_pull(task_ids='add_steps', key='return_value')[" 173 | + str(last_step) 174 | + "] }}", 175 | aws_conn_id="aws_default", 176 | dag=dag, 177 | ) 178 | 179 | # Terminate the EMR cluster 180 | terminate_emr_cluster = EmrTerminateJobFlowOperator( 181 | task_id="terminate_emr_cluster", 182 | job_flow_id="{{ task_instance.xcom_pull(task_ids='create_emr_cluster', key='return_value') }}", 183 | aws_conn_id="aws_default", 184 | dag=dag, 185 | ) 186 | 187 | end_data_pipeline = DummyOperator(task_id="end_data_pipeline", dag=dag) 188 | 189 | start_data_pipeline >> [data_to_s3, script_to_s3] >> create_emr_cluster 190 | create_emr_cluster >> step_adder >> step_checker >> terminate_emr_cluster 191 | terminate_emr_cluster >> end_data_pipeline 192 | -------------------------------------------------------------------------------- /docker-compose-LocalExecutor.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | postgres: 4 | image: postgres:9.6 5 | environment: 6 | - POSTGRES_USER=airflow 7 | - POSTGRES_PASSWORD=airflow 8 | - POSTGRES_DB=airflow 9 | logging: 10 | options: 11 | max-size: 10m 12 | max-file: "3" 13 | volumes: 14 | - ./setup/raw_input_data:/data 15 | - ./temp:/temp 16 | ports: 17 | - "5432:5432" 18 | 19 | webserver: 20 | image: puckel/docker-airflow:1.10.9 21 | restart: always 22 | depends_on: 23 | - postgres 24 | environment: 25 | - LOAD_EX=n 26 | - EXECUTOR=Local 27 | - AIRFLOW_CONN_POSTGRES_DEFAULT=postgres://airflow:airflow@postgres:5432/airflow 28 | - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= 29 | - AWS_SHARED_CREDENTIALS_FILE=/usr/local/airflow/.aws/credentials 30 | logging: 31 | options: 32 | max-size: 10m 33 | max-file: "3" 34 | volumes: 35 | - ./dags:/usr/local/airflow/dags 36 | # - ./plugins:/usr/local/airflow/plugins 37 | # for linux based systems 38 | - ./requirements.txt:/requirements.txt 39 | - ~/.aws:/usr/local/airflow/.aws/ 40 | - ./temp:/temp 41 | - ./setup/raw_input_data:/data 42 | ports: 43 | - "8080:8080" 44 | command: webserver 45 | healthcheck: 46 | test: 47 | ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] 48 | interval: 30s 49 | timeout: 30s 50 | retries: 3 51 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto3===1.8.4 2 | paramiko===2.4.2 -------------------------------------------------------------------------------- /script/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # User-provided configuration must always be respected. 4 | # 5 | # Therefore, this script must only derives Airflow AIRFLOW__ variables from other variables 6 | # when the user did not provide their own configuration. 7 | 8 | TRY_LOOP="20" 9 | 10 | # Global defaults and back-compat 11 | : "${AIRFLOW_HOME:="/usr/local/airflow"}" 12 | : "${AIRFLOW__CORE__FERNET_KEY:=${FERNET_KEY:=$(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)")}}" 13 | : "${AIRFLOW__CORE__EXECUTOR:=${EXECUTOR:-Sequential}Executor}" 14 | 15 | # Load DAGs examples (default: Yes) 16 | if [[ -z "$AIRFLOW__CORE__LOAD_EXAMPLES" && "${LOAD_EX:=n}" == n ]]; then 17 | AIRFLOW__CORE__LOAD_EXAMPLES=False 18 | fi 19 | 20 | export \ 21 | AIRFLOW_HOME \ 22 | AIRFLOW__CORE__EXECUTOR \ 23 | AIRFLOW__CORE__FERNET_KEY \ 24 | AIRFLOW__CORE__LOAD_EXAMPLES \ 25 | 26 | # Install custom python package if requirements.txt is present 27 | if [ -e "/requirements.txt" ]; then 28 | $(command -v pip) install --user -r /requirements.txt 29 | fi 30 | 31 | wait_for_port() { 32 | local name="$1" host="$2" port="$3" 33 | local j=0 34 | while ! nc -z "$host" "$port" >/dev/null 2>&1 < /dev/null; do 35 | j=$((j+1)) 36 | if [ $j -ge $TRY_LOOP ]; then 37 | echo >&2 "$(date) - $host:$port still not reachable, giving up" 38 | exit 1 39 | fi 40 | echo "$(date) - waiting for $name... $j/$TRY_LOOP" 41 | sleep 5 42 | done 43 | } 44 | 45 | # Other executors than SequentialExecutor drive the need for an SQL database, here PostgreSQL is used 46 | if [ "$AIRFLOW__CORE__EXECUTOR" != "SequentialExecutor" ]; then 47 | # Check if the user has provided explicit Airflow configuration concerning the database 48 | if [ -z "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" ]; then 49 | # Default values corresponding to the default compose files 50 | : "${POSTGRES_HOST:="postgres"}" 51 | : "${POSTGRES_PORT:="5432"}" 52 | : "${POSTGRES_USER:="airflow"}" 53 | : "${POSTGRES_PASSWORD:="airflow"}" 54 | : "${POSTGRES_DB:="airflow"}" 55 | : "${POSTGRES_EXTRAS:-""}" 56 | 57 | AIRFLOW__CORE__SQL_ALCHEMY_CONN="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}${POSTGRES_EXTRAS}" 58 | export AIRFLOW__CORE__SQL_ALCHEMY_CONN 59 | 60 | # Check if the user has provided explicit Airflow configuration for the broker's connection to the database 61 | if [ "$AIRFLOW__CORE__EXECUTOR" = "CeleryExecutor" ]; then 62 | AIRFLOW__CELERY__RESULT_BACKEND="db+postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}${POSTGRES_EXTRAS}" 63 | export AIRFLOW__CELERY__RESULT_BACKEND 64 | fi 65 | else 66 | if [[ "$AIRFLOW__CORE__EXECUTOR" == "CeleryExecutor" && -z "$AIRFLOW__CELERY__RESULT_BACKEND" ]]; then 67 | >&2 printf '%s\n' "FATAL: if you set AIRFLOW__CORE__SQL_ALCHEMY_CONN manually with CeleryExecutor you must also set AIRFLOW__CELERY__RESULT_BACKEND" 68 | exit 1 69 | fi 70 | 71 | # Derive useful variables from the AIRFLOW__ variables provided explicitly by the user 72 | POSTGRES_ENDPOINT=$(echo -n "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" | cut -d '/' -f3 | sed -e 's,.*@,,') 73 | POSTGRES_HOST=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f1) 74 | POSTGRES_PORT=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f2) 75 | fi 76 | 77 | wait_for_port "Postgres" "$POSTGRES_HOST" "$POSTGRES_PORT" 78 | fi 79 | 80 | # CeleryExecutor drives the need for a Celery broker, here Redis is used 81 | if [ "$AIRFLOW__CORE__EXECUTOR" = "CeleryExecutor" ]; then 82 | # Check if the user has provided explicit Airflow configuration concerning the broker 83 | if [ -z "$AIRFLOW__CELERY__BROKER_URL" ]; then 84 | # Default values corresponding to the default compose files 85 | : "${REDIS_PROTO:="redis://"}" 86 | : "${REDIS_HOST:="redis"}" 87 | : "${REDIS_PORT:="6379"}" 88 | : "${REDIS_PASSWORD:=""}" 89 | : "${REDIS_DBNUM:="1"}" 90 | 91 | # When Redis is secured by basic auth, it does not handle the username part of basic auth, only a token 92 | if [ -n "$REDIS_PASSWORD" ]; then 93 | REDIS_PREFIX=":${REDIS_PASSWORD}@" 94 | else 95 | REDIS_PREFIX= 96 | fi 97 | 98 | AIRFLOW__CELERY__BROKER_URL="${REDIS_PROTO}${REDIS_PREFIX}${REDIS_HOST}:${REDIS_PORT}/${REDIS_DBNUM}" 99 | export AIRFLOW__CELERY__BROKER_URL 100 | else 101 | # Derive useful variables from the AIRFLOW__ variables provided explicitly by the user 102 | REDIS_ENDPOINT=$(echo -n "$AIRFLOW__CELERY__BROKER_URL" | cut -d '/' -f3 | sed -e 's,.*@,,') 103 | REDIS_HOST=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f1) 104 | REDIS_PORT=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f2) 105 | fi 106 | 107 | wait_for_port "Redis" "$REDIS_HOST" "$REDIS_PORT" 108 | fi 109 | 110 | case "$1" in 111 | webserver) 112 | airflow initdb 113 | if [ "$AIRFLOW__CORE__EXECUTOR" = "LocalExecutor" ] || [ "$AIRFLOW__CORE__EXECUTOR" = "SequentialExecutor" ]; then 114 | # With the "Local" and "Sequential" executors it should all run in one container. 115 | airflow scheduler & 116 | fi 117 | exec airflow webserver 118 | ;; 119 | worker|scheduler) 120 | # Give the webserver time to run initdb. 121 | sleep 10 122 | exec airflow "$@" 123 | ;; 124 | flower) 125 | sleep 10 126 | exec airflow "$@" 127 | ;; 128 | version) 129 | exec airflow "$@" 130 | ;; 131 | *) 132 | # The command is something like bash, not an airflow subcommand. Just run it in the right environment. 133 | exec "$@" 134 | ;; 135 | esac 136 | -------------------------------------------------------------------------------- /workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | ci: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v1 16 | - run: docker build -t "${PWD##*/}" . 17 | - run: docker run "${PWD##*/}" python -V 18 | - run: docker run "${PWD##*/}" version 19 | --------------------------------------------------------------------------------