├── .env ├── .gitignore ├── requirements.txt ├── README.md ├── Dockerfile ├── dags ├── classic_flow.py └── task_flow.py ├── LICENSE └── docker-compose.yaml /.env: -------------------------------------------------------------------------------- 1 | AIRFLOW_UID=1000 2 | AIRFLOW_GID=0 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /venv/ 2 | /logs/ 3 | /.idea/ 4 | 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | lxml 2 | beautifulsoup4 3 | apache-airflow==2.9.2 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # airflow-docker 2 | Airflow for docker deploy(only for study) 3 | 4 | # version 5 | 6 | ## System version 7 | 1.0.0 8 | 9 | ## Airflow version 10 | 2.9.2 11 | 12 | # Pre-requisites: 13 | 14 | * docker 15 | * docker-compose 16 | * Python >= 3.10 17 | 18 | **IMPORTANT:** If you're using Linux, remove the default docker package and install the official packages using official 19 | docker site documentation. 20 | 21 | # Getting started 22 | 23 | - Git pull 24 | 25 | - Then execute: `docker compose up` 26 | 27 | - **WAIT FOR THE COMPLETE PROCESS IS FINISHED!** 28 | 29 | - You should be able to access the Airflow web interface using port **8092** 30 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/airflow:2.9.2 2 | 3 | # INSTALL BASE packages 4 | 5 | USER root 6 | RUN apt-get update \ 7 | && apt-get install -y --no-install-recommends \ 8 | vim \ 9 | && apt-get autoremove -yqq --purge \ 10 | && apt-get clean \ 11 | && rm -rf /var/lib/apt/lists/* 12 | 13 | RUN mkdir -p /opt/airflow/logs/scheduler/2024-06-28 \ 14 | ; chown -R airflow /opt/airflow/logs \ 15 | ; chmod 777 -R /opt/airflow 16 | 17 | USER airflow 18 | 19 | # COPY package references file to the container 20 | COPY requirements.txt / 21 | # INSTALL AIRFLOW Python packages 22 | RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" -r /requirements.txt 23 | 24 | COPY --chown=airflow:root test_dag.py /opt/airflow/dags 25 | 26 | ENV AIRFLOW__CORE__LOAD_EXAMPLES=false 27 | 28 | -------------------------------------------------------------------------------- /dags/classic_flow.py: -------------------------------------------------------------------------------- 1 | 2 | import textwrap 3 | from datetime import datetime, timedelta 4 | 5 | # The DAG object; we'll need this to instantiate a DAG 6 | from airflow.models.dag import DAG 7 | 8 | # Operators; we need this to operate! 9 | from airflow.operators.bash import BashOperator 10 | from airflow.operators.python import PythonOperator, BranchPythonOperator 11 | from airflow.operators.empty import EmptyOperator 12 | from airflow.utils.trigger_rule import TriggerRule 13 | 14 | 15 | 16 | 17 | 18 | with DAG( 19 | "cl_pipeline", 20 | # These args will get passed on to each operator 21 | # You can override them on a per-task basis during operator initialization 22 | default_args={ 23 | "depends_on_past": False, 24 | "email": ["andregarciacarneiro@gmail.com"], 25 | "email_on_failure": False, 26 | "email_on_retry": False, 27 | # "retries": 1, 28 | # "retry_delay": timedelta(minutes=5), 29 | 30 | }, 31 | description="A simple tutorial DAG", 32 | schedule=None, 33 | start_date=datetime(2021, 1, 1), 34 | catchup=False, 35 | tags=["branch", "example"], 36 | ) as dag: 37 | 38 | transform1 = PythonOperator( 39 | task_id='transform1', 40 | do_xcom_push=False, 41 | python_callable=transform1 42 | ) 43 | 44 | triage = BranchPythonOperator( 45 | task_id='triage', 46 | python_callable=triage, 47 | ) 48 | 49 | extract2 = PythonOperator( 50 | task_id='extract2', 51 | python_callable=extract2 52 | ) 53 | 54 | extract1 = PythonOperator( 55 | task_id='extract1', 56 | python_callable=extract1 57 | ) 58 | 59 | # transform1 = EmptyOperator(task_id='transform1') 60 | # transform2 = EmptyOperator(task_id='transform2') 61 | start = EmptyOperator(task_id='start', trigger_rule=TriggerRule.ALL_DONE) 62 | end = EmptyOperator(task_id='end', trigger_rule=TriggerRule.ALL_DONE) 63 | 64 | start >> [extract1, extract2] >> triage >> [transform1, transform2] >> end 65 | -------------------------------------------------------------------------------- /dags/task_flow.py: -------------------------------------------------------------------------------- 1 | import json 2 | from datetime import datetime 3 | 4 | from airflow import DAG 5 | from airflow.decorators import task 6 | from airflow.operators.bash import BashOperator 7 | from airflow.operators.empty import EmptyOperator 8 | 9 | # A DAG represents a workflow, a collection of tasks 10 | with DAG( 11 | dag_id="tf_pipeline", 12 | start_date=datetime(2022, 1, 1), 13 | schedule=None 14 | ) as dag: 15 | 16 | start = EmptyOperator(task_id="start", trigger_rule="all_done") 17 | end = EmptyOperator(task_id="end") 18 | 19 | @task(task_id="load") 20 | def load(): 21 | print("Loading using task flow") 22 | 23 | @task(task_id="transform3") 24 | def transform3(*, ti): 25 | print(f"Transforming3 using task flow {ti.xcom_pull('xc')}") 26 | 27 | @task(task_id="transform2") 28 | def transform2(*, ti): 29 | print(f"Transforming3 using task flow {ti.xcom_pull('xc')}") 30 | 31 | @task(task_id="transform1") 32 | def transform1(*, ti): 33 | print(f"Transforming3 using task flow {ti.xcom_pull('xc')}") 34 | 35 | @task(task_id="extract3") 36 | def extract3(*, ti): 37 | json_data = '{"kind": 3,"data": "Test3"}' 38 | ti.xcom_push('xc',json_data) 39 | return json_data 40 | 41 | @task(task_id="extract2") 42 | def extract2(*, ti): 43 | json_data = '{"kind": 2,"data": "Test2"}' 44 | ti.xcom_push('xc', json_data) 45 | return json_data 46 | 47 | @task(task_id="extract1") 48 | def extract1(*, ti): 49 | json_data = '{"kind": 1,"data": "Test1"}' 50 | ti.xcom_push('xc', json_data) 51 | return json_data 52 | 53 | @task() 54 | def airflow(): 55 | print("airflow") 56 | 57 | 58 | @task.branch(task_id='triage') 59 | def triage(*, ti): 60 | all_data = ti.xcom_pull('xc') 61 | if isinstance(all_data, str) and len(all_data) > 0: 62 | all_data = json.loads(all_data) 63 | 64 | print(f"JSONDATA: {str(all_data)}") 65 | transformer = { 66 | 'kind1': 'transform1', 67 | 'kind2': 'transform2', 68 | 'kind3': 'transform3' 69 | } 70 | 71 | for x in all_data: 72 | kind = x['kind'] 73 | yield transformer[kind] 74 | 75 | # Set dependencies between tasks 76 | start >> [extract1(), extract2(), extract3()] >> triage() >> [transform1(), transform2(), transform3()] >> load() >> end 77 | # start >> [extract1(), extract2(), extract3()] >> end -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | # 18 | 19 | # Basic Airflow cluster configuration for CeleryExecutor with Redis and PostgreSQL. 20 | # 21 | # WARNING: This configuration is for local development. Do not use it in a production deployment. 22 | # 23 | # This configuration supports basic configuration using environment variables or an .env file 24 | # The following variables are supported: 25 | # 26 | # AIRFLOW_IMAGE_NAME - Docker image name used to run Airflow. 27 | # Default: apache/airflow:2.9.2 28 | # AIRFLOW_UID - User ID in Airflow containers 29 | # Default: 50000 30 | # AIRFLOW_PROJ_DIR - Base path to which all the files will be volumed. 31 | # Default: . 32 | # Those configurations are useful mostly in case of standalone testing/running Airflow in test/try-out mode 33 | # 34 | # _AIRFLOW_WWW_USER_USERNAME - Username for the administrator account (if requested). 35 | # Default: airflow 36 | # _AIRFLOW_WWW_USER_PASSWORD - Password for the administrator account (if requested). 37 | # Default: airflow 38 | # _PIP_ADDITIONAL_REQUIREMENTS - Additional PIP requirements to add when starting all containers. 39 | # Use this option ONLY for quick checks. Installing requirements at container 40 | # startup is done EVERY TIME the service is started. 41 | # A better way is to build a custom image or extend the official image 42 | # as described in https://airflow.apache.org/docs/docker-stack/build.html. 43 | # Default: '' 44 | # 45 | # Feel free to modify this file to suit your needs. 46 | --- 47 | x-airflow-common: 48 | &airflow-common 49 | # In order to add custom dependencies or upgrade provider packages you can use your extended image. 50 | # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml 51 | # and uncomment the "build" line below, Then run `docker-compose build` to build the images. 52 | # image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.9.2} 53 | image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.9.2} 54 | # build: . 55 | environment: 56 | &airflow-common-env 57 | AIRFLOW__CORE__EXECUTOR: CeleryExecutor 58 | AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@airflow-postgres/airflow 59 | AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@airflow-postgres/airflow 60 | AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 61 | AIRFLOW__CORE__FERNET_KEY: '' 62 | AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' 63 | AIRFLOW__CORE__LOAD_EXAMPLES: 'false' 64 | AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session' 65 | # yamllint disable rule:line-length 66 | # Use simple http server on scheduler for health checks 67 | # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server 68 | # yamllint enable rule:line-length 69 | AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' 70 | # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks 71 | # for other purpose (development, test and especially production usage) build/extend Airflow image. 72 | _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} 73 | # The following line can be used to set a custom config file, stored in the local config folder 74 | # If you want to use it, outcomment it and replace airflow.cfg with the name of your config file 75 | # AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg' 76 | AIRFLOW_UID: 1000 77 | volumes: 78 | - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags 79 | - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs 80 | - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config 81 | - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins 82 | user: "${AIRFLOW_UID:-1000}:0" 83 | depends_on: 84 | &airflow-common-depends-on 85 | redis: 86 | condition: service_healthy 87 | postgres: 88 | condition: service_healthy 89 | networks: 90 | - ramtricks 91 | 92 | services: 93 | postgres: 94 | container_name: airflow-postgres 95 | image: postgres:13 96 | environment: 97 | POSTGRES_USER: airflow 98 | POSTGRES_PASSWORD: airflow 99 | POSTGRES_DB: airflow 100 | volumes: 101 | - /storage/postgres-db-volume:/var/lib/postgresql/data 102 | healthcheck: 103 | test: ["CMD", "pg_isready", "-U", "airflow"] 104 | interval: 10s 105 | retries: 5 106 | start_period: 5s 107 | restart: always 108 | ports: 109 | - '5434:5432' 110 | networks: 111 | - ramtricks 112 | 113 | 114 | 115 | redis: 116 | # Redis is limited to 7.2-bookworm due to licencing change 117 | # https://redis.io/blog/redis-adopts-dual-source-available-licensing/ 118 | container_name: airflow-redis 119 | image: redis:7.2-bookworm 120 | expose: 121 | - 6379 122 | healthcheck: 123 | test: ["CMD", "redis-cli", "ping"] 124 | interval: 10s 125 | timeout: 30s 126 | retries: 50 127 | start_period: 30s 128 | restart: always 129 | networks: 130 | - ramtricks 131 | 132 | airflow-webserver: 133 | <<: *airflow-common 134 | container_name: airflow-webserver 135 | command: webserver 136 | ports: 137 | - "8095:8080" 138 | healthcheck: 139 | test: ["CMD", "curl", "--fail", "http://localhost:8080/health"] 140 | interval: 30s 141 | timeout: 10s 142 | retries: 5 143 | start_period: 30s 144 | restart: always 145 | depends_on: 146 | <<: *airflow-common-depends-on 147 | airflow-init: 148 | condition: service_completed_successfully 149 | 150 | airflow-scheduler: 151 | <<: *airflow-common 152 | container_name: airflow-scheduler 153 | command: scheduler 154 | healthcheck: 155 | test: ["CMD", "curl", "--fail", "http://localhost:8974/health"] 156 | interval: 30s 157 | timeout: 10s 158 | retries: 5 159 | start_period: 30s 160 | restart: always 161 | depends_on: 162 | <<: *airflow-common-depends-on 163 | airflow-init: 164 | condition: service_completed_successfully 165 | 166 | airflow-worker: 167 | <<: *airflow-common 168 | container_name: airflow-worker1 169 | command: celery worker 170 | healthcheck: 171 | # yamllint disable rule:line-length 172 | test: 173 | - "CMD-SHELL" 174 | - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' 175 | interval: 30s 176 | timeout: 10s 177 | retries: 5 178 | start_period: 30s 179 | environment: 180 | <<: *airflow-common-env 181 | # Required to handle warm shutdown of the celery workers properly 182 | # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation 183 | DUMB_INIT_SETSID: "0" 184 | restart: always 185 | depends_on: 186 | <<: *airflow-common-depends-on 187 | airflow-init: 188 | condition: service_completed_successfully 189 | 190 | # airflow-triggerer: 191 | # <<: *airflow-common 192 | # command: triggerer 193 | # healthcheck: 194 | # test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'] 195 | # interval: 30s 196 | # timeout: 10s 197 | # retries: 5 198 | # start_period: 30s 199 | # restart: always 200 | # depends_on: 201 | # <<: *airflow-common-depends-on 202 | # airflow-init: 203 | # condition: service_completed_successfully 204 | 205 | airflow-init: 206 | <<: *airflow-common 207 | entrypoint: /bin/bash 208 | # yamllint disable rule:line-length 209 | command: 210 | - -c 211 | - | 212 | if [[ -z "${AIRFLOW_UID}" ]]; then 213 | echo 214 | echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m" 215 | echo "If you are on Linux, you SHOULD follow the instructions below to set " 216 | echo "AIRFLOW_UID environment variable, otherwise files will be owned by root." 217 | echo "For other operating systems you can get rid of the warning with manually created .env file:" 218 | echo " See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user" 219 | echo 220 | fi 221 | one_meg=1048576 222 | mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg)) 223 | cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat) 224 | disk_available=$$(df / | tail -1 | awk '{print $$4}') 225 | warning_resources="false" 226 | if (( mem_available < 4000 )) ; then 227 | echo 228 | echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m" 229 | echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))" 230 | echo 231 | warning_resources="true" 232 | fi 233 | if (( cpus_available < 2 )); then 234 | echo 235 | echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m" 236 | echo "At least 2 CPUs recommended. You have $${cpus_available}" 237 | echo 238 | warning_resources="true" 239 | fi 240 | if (( disk_available < one_meg * 10 )); then 241 | echo 242 | echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m" 243 | echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))" 244 | echo 245 | warning_resources="true" 246 | fi 247 | if [[ $${warning_resources} == "true" ]]; then 248 | echo 249 | echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m" 250 | echo "Please follow the instructions to increase amount of resources available:" 251 | echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin" 252 | echo 253 | fi 254 | mkdir -p /sources/logs /sources/dags /sources/plugins 255 | chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} 256 | exec /entrypoint airflow version 257 | # yamllint enable rule:line-length 258 | environment: 259 | <<: *airflow-common-env 260 | _AIRFLOW_DB_MIGRATE: 'true' 261 | _AIRFLOW_WWW_USER_CREATE: 'true' 262 | _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} 263 | _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} 264 | _PIP_ADDITIONAL_REQUIREMENTS: '' 265 | user: "0:0" 266 | volumes: 267 | - ${AIRFLOW_PROJ_DIR:-.}:/sources 268 | 269 | airflow-cli: 270 | <<: *airflow-common 271 | profiles: 272 | - debug 273 | environment: 274 | <<: *airflow-common-env 275 | CONNECTION_CHECK_MAX_COUNT: "0" 276 | # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252 277 | command: 278 | - bash 279 | - -c 280 | - airflow 281 | 282 | # You can enable flower by adding "--profile flower" option e.g. docker-compose --profile flower up 283 | # or by explicitly targeted on the command line e.g. docker-compose up flower. 284 | # See: https://docs.docker.com/compose/profiles/ 285 | flower: 286 | <<: *airflow-common 287 | command: celery flower 288 | profiles: 289 | - flower 290 | ports: 291 | - "5555:5555" 292 | healthcheck: 293 | test: ["CMD", "curl", "--fail", "http://localhost:5555/"] 294 | interval: 30s 295 | timeout: 10s 296 | retries: 5 297 | start_period: 30s 298 | restart: always 299 | depends_on: 300 | <<: *airflow-common-depends-on 301 | airflow-init: 302 | condition: service_completed_successfully 303 | 304 | volumes: 305 | postgres-db-volume: 306 | 307 | networks: 308 | ramtricks: 309 | driver: bridge --------------------------------------------------------------------------------