├── .github └── workflows │ └── build_and_deploy.yaml ├── .gitignore ├── README.md ├── chart └── airflow │ ├── .helmignore │ ├── CHANGELOG.txt │ ├── Chart.lock │ ├── Chart.yaml │ ├── INSTALL │ ├── LICENSE │ ├── NOTICE │ ├── README.md │ ├── UPDATING.rst │ ├── charts │ └── postgresql │ │ ├── .helmignore │ │ ├── Chart.lock │ │ ├── Chart.yaml │ │ ├── README.md │ │ ├── charts │ │ └── common │ │ │ ├── .helmignore │ │ │ ├── Chart.yaml │ │ │ ├── README.md │ │ │ ├── templates │ │ │ ├── _affinities.tpl │ │ │ ├── _capabilities.tpl │ │ │ ├── _errors.tpl │ │ │ ├── _images.tpl │ │ │ ├── _ingress.tpl │ │ │ ├── _labels.tpl │ │ │ ├── _names.tpl │ │ │ ├── _secrets.tpl │ │ │ ├── _storage.tpl │ │ │ ├── _tplvalues.tpl │ │ │ ├── _utils.tpl │ │ │ ├── _warnings.tpl │ │ │ └── validations │ │ │ │ ├── _cassandra.tpl │ │ │ │ ├── _mariadb.tpl │ │ │ │ ├── _mongodb.tpl │ │ │ │ ├── _postgresql.tpl │ │ │ │ ├── _redis.tpl │ │ │ │ └── _validations.tpl │ │ │ └── values.yaml │ │ ├── ci │ │ ├── commonAnnotations.yaml │ │ ├── default-values.yaml │ │ └── shmvolume-disabled-values.yaml │ │ ├── files │ │ ├── README.md │ │ ├── conf.d │ │ │ └── README.md │ │ └── docker-entrypoint-initdb.d │ │ │ └── README.md │ │ ├── templates │ │ ├── NOTES.txt │ │ ├── _helpers.tpl │ │ ├── configmap.yaml │ │ ├── extended-config-configmap.yaml │ │ ├── extra-list.yaml │ │ ├── initialization-configmap.yaml │ │ ├── metrics-configmap.yaml │ │ ├── metrics-svc.yaml │ │ ├── networkpolicy.yaml │ │ ├── podsecuritypolicy.yaml │ │ ├── prometheusrule.yaml │ │ ├── role.yaml │ │ ├── rolebinding.yaml │ │ ├── secrets.yaml │ │ ├── serviceaccount.yaml │ │ ├── servicemonitor.yaml │ │ ├── statefulset-readreplicas.yaml │ │ ├── statefulset.yaml │ │ ├── svc-headless.yaml │ │ ├── svc-read.yaml │ │ ├── svc.yaml │ │ └── tls-secrets.yaml │ │ ├── values.schema.json │ │ └── values.yaml │ ├── dockerfiles │ ├── README.md │ ├── pgbouncer-exporter │ │ ├── Dockerfile │ │ └── build_and_push.sh │ ├── pgbouncer │ │ ├── Dockerfile │ │ └── build_and_push.sh │ └── statsd-exporter │ │ ├── Dockerfile │ │ ├── build_and_push.sh │ │ └── mappings.yml │ ├── files │ └── pod-template-file.kubernetes-helm-yaml │ ├── templates │ ├── NOTES.txt │ ├── _helpers.yaml │ ├── check-values.yaml │ ├── cleanup │ │ ├── cleanup-cronjob.yaml │ │ └── cleanup-serviceaccount.yaml │ ├── configmaps │ │ ├── configmap.yaml │ │ ├── extra-configmaps.yaml │ │ ├── statsd-configmap.yaml │ │ └── webserver-configmap.yaml │ ├── dags-persistent-volume-claim.yaml │ ├── flower │ │ ├── flower-deployment.yaml │ │ ├── flower-ingress.yaml │ │ ├── flower-networkpolicy.yaml │ │ ├── flower-service.yaml │ │ └── flower-serviceaccount.yaml │ ├── jobs │ │ ├── create-user-job-serviceaccount.yaml │ │ ├── create-user-job.yaml │ │ ├── migrate-database-job-serviceaccount.yaml │ │ └── migrate-database-job.yaml │ ├── limitrange.yaml │ ├── logs-persistent-volume-claim.yaml │ ├── pgbouncer │ │ ├── pgbouncer-deployment.yaml │ │ ├── pgbouncer-networkpolicy.yaml │ │ ├── pgbouncer-poddisruptionbudget.yaml │ │ ├── pgbouncer-service.yaml │ │ └── pgbouncer-serviceaccount.yaml │ ├── rbac │ │ ├── pod-cleanup-role.yaml │ │ ├── pod-cleanup-rolebinding.yaml │ │ ├── pod-launcher-role.yaml │ │ ├── pod-launcher-rolebinding.yaml │ │ ├── pod-log-reader-role.yaml │ │ ├── pod-log-reader-rolebinding.yaml │ │ └── security-context-constraint-rolebinding.yaml │ ├── redis │ │ ├── redis-networkpolicy.yaml │ │ ├── redis-service.yaml │ │ ├── redis-serviceaccount.yaml │ │ └── redis-statefulset.yaml │ ├── resourcequota.yaml │ ├── scheduler │ │ ├── scheduler-deployment.yaml │ │ ├── scheduler-networkpolicy.yaml │ │ ├── scheduler-poddisruptionbudget.yaml │ │ ├── scheduler-service.yaml │ │ └── scheduler-serviceaccount.yaml │ ├── secrets │ │ ├── elasticsearch-secret.yaml │ │ ├── extra-secrets.yaml │ │ ├── fernetkey-secret.yaml │ │ ├── flower-secret.yaml │ │ ├── kerberos-keytab-secret.yaml │ │ ├── metadata-connection-secret.yaml │ │ ├── pgbouncer-certificates-secret.yaml │ │ ├── pgbouncer-config-secret.yaml │ │ ├── pgbouncer-stats-secret.yaml │ │ ├── redis-secrets.yaml │ │ ├── registry-secret.yaml │ │ ├── result-backend-connection-secret.yaml │ │ └── webserver-secret-key-secret.yaml │ ├── statsd │ │ ├── statsd-deployment.yaml │ │ ├── statsd-networkpolicy.yaml │ │ ├── statsd-service.yaml │ │ └── statsd-serviceaccount.yaml │ ├── triggerer │ │ ├── triggerer-deployment.yaml │ │ └── triggerer-serviceaccount.yaml │ ├── webserver │ │ ├── webserver-deployment.yaml │ │ ├── webserver-ingress.yaml │ │ ├── webserver-networkpolicy.yaml │ │ ├── webserver-service.yaml │ │ └── webserver-serviceaccount.yaml │ └── workers │ │ ├── worker-deployment.yaml │ │ ├── worker-kedaautoscaler.yaml │ │ ├── worker-networkpolicy.yaml │ │ ├── worker-service.yaml │ │ └── worker-serviceaccount.yaml │ ├── values.schema.json │ ├── values.yaml │ └── values_schema.schema.json ├── docs ├── package.json ├── src │ ├── .vuepress │ │ ├── components │ │ │ ├── Foo │ │ │ │ └── Bar.vue │ │ │ ├── OtherComponent.vue │ │ │ └── demo-component.vue │ │ ├── config.js │ │ ├── enhanceApp.js │ │ ├── public │ │ │ └── robots.txt │ │ └── styles │ │ │ ├── index.styl │ │ │ └── palette.styl │ ├── architecture │ │ ├── 01-overview │ │ │ ├── README.md │ │ │ └── img.png │ │ ├── 02-dag-directory │ │ │ └── README.md │ │ ├── 03-scheduler │ │ │ └── README.md │ │ ├── 04-executor │ │ │ ├── 01-sequential-executor │ │ │ │ └── README.md │ │ │ ├── 02-local-executor │ │ │ │ └── README.md │ │ │ ├── 03-celery-executor │ │ │ │ ├── README.md │ │ │ │ └── img.png │ │ │ ├── 04-kubernetes-executor │ │ │ │ ├── README.md │ │ │ │ └── img.png │ │ │ └── README.md │ │ ├── 05-meta-database │ │ │ ├── README.md │ │ │ └── airflow_meta_db_erd.png │ │ ├── 06-webserver │ │ │ ├── README.md │ │ │ └── img.png │ │ └── README.md │ ├── dags │ │ ├── 01-writing-various-tasks-dependencies │ │ │ ├── 01-simple-task-dependency │ │ │ │ ├── README.md │ │ │ │ ├── image-20220122121951658.png │ │ │ │ └── image-20220122122006564.png │ │ │ ├── 02-parallel-tasks-flow │ │ │ │ ├── README.md │ │ │ │ ├── image-20220122115223321.png │ │ │ │ └── image-20220122122532097.png │ │ │ ├── 03-parallel-task-list-flow │ │ │ │ ├── README.md │ │ │ │ ├── image-20220122150241425.png │ │ │ │ └── image-20220122150332463.png │ │ │ ├── 04-conditional-task-flow-with-branch │ │ │ │ ├── README.md │ │ │ │ ├── image-20220122171121697.png │ │ │ │ └── image-20220123003919600.png │ │ │ ├── 05-conditional-task-flow-with-depends-on-past │ │ │ │ ├── README.md │ │ │ │ ├── image-20220122220345864.png │ │ │ │ ├── image-20220122225639740.png │ │ │ │ └── image-20220123003635284.png │ │ │ ├── 06-conditional-task-flow-with-wait-for-downstream │ │ │ │ ├── README.md │ │ │ │ ├── image-20220122225546239.png │ │ │ │ ├── image-20220122225611854.png │ │ │ │ └── image-20220123003728946.png │ │ │ ├── 07-conditional-task-flow-with-trigger-rule │ │ │ │ ├── README.md │ │ │ │ ├── image-20220123003032553.png │ │ │ │ └── image-20220123003131411.png │ │ │ ├── 08-subdag │ │ │ │ ├── README.md │ │ │ │ ├── image-20220123154556951.png │ │ │ │ ├── image-20220123154746623.png │ │ │ │ ├── image-20220123154845833.png │ │ │ │ ├── image-20220123154859276.png │ │ │ │ └── image-20220123154910866.png │ │ │ └── README.md │ │ ├── 02-dependencies-between-dags │ │ │ ├── 01-trigger-dagrun │ │ │ │ ├── README.md │ │ │ │ ├── image-20220123162512663.png │ │ │ │ ├── image-20220123162839172.png │ │ │ │ ├── image-20220123162936827.png │ │ │ │ ├── image-20220123163031694.png │ │ │ │ ├── image-20220123163048155.png │ │ │ │ ├── image-20220123163102483.png │ │ │ │ └── image-20220123163941903.png │ │ │ ├── 02-waiting-tasks-in-another-dagrun │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_3.png │ │ │ │ ├── img_4.png │ │ │ │ ├── img_5.png │ │ │ │ └── img_6.png │ │ │ └── README.md │ │ ├── 03-visualization-and-documentation-dags │ │ │ ├── 01-task-groups │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ └── img_1.png │ │ │ ├── 02-labels │ │ │ │ ├── README.md │ │ │ │ └── img.png │ │ │ ├── 03-documentation │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ └── img_2.png │ │ │ ├── 04-save-dag-image │ │ │ │ ├── README.md │ │ │ │ └── output.png │ │ │ └── README.md │ │ ├── 04-using-various-operators │ │ │ ├── 01-bash-operator │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ └── img_3.png │ │ │ ├── 02-python-operator │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_3.png │ │ │ │ ├── img_4.png │ │ │ │ └── img_5.png │ │ │ ├── 03-python-virtualenv-operator │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ └── img_3.png │ │ │ ├── 04-branch-python-operator │ │ │ │ ├── README.md │ │ │ │ ├── image-20220122171121697.png │ │ │ │ └── image-20220123003919600.png │ │ │ ├── 05-dummy-operator │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ └── img_2.png │ │ │ ├── 06-email-operator │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_3.png │ │ │ │ ├── img_4.png │ │ │ │ └── img_5.png │ │ │ ├── 07-custom-operator │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ └── img_2.png │ │ │ └── README.md │ │ ├── 05-etc-features │ │ │ ├── 01-xcoms │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_3.png │ │ │ │ └── img_4.png │ │ │ ├── 02-variables │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_3.png │ │ │ │ ├── img_4.png │ │ │ │ ├── img_5.png │ │ │ │ ├── img_7.png │ │ │ │ ├── img_8.png │ │ │ │ └── img_9.png │ │ │ ├── 03-connections-and-hooks │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_3.png │ │ │ │ ├── img_4.png │ │ │ │ ├── img_5.png │ │ │ │ └── img_6.png │ │ │ ├── 04-templates │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_3.png │ │ │ │ ├── img_4.png │ │ │ │ └── img_5.png │ │ │ └── README.md │ │ ├── 06-taskflow-api │ │ │ ├── 01-concept │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_3.png │ │ │ │ ├── img_4.png │ │ │ │ └── img_5.png │ │ │ ├── 02-custom-dependencies │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_3.png │ │ │ │ └── img_4.png │ │ │ ├── 03-with-operators │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_3.png │ │ │ │ └── img_4.png │ │ │ ├── 04-virtualenv │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ └── img_2.png │ │ │ ├── 05-docker │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ └── img_2.png │ │ │ └── README.md │ │ ├── 07-providers │ │ │ ├── README.md │ │ │ ├── img.png │ │ │ ├── img_1.png │ │ │ ├── img_2.png │ │ │ ├── img_3.png │ │ │ ├── img_4.png │ │ │ ├── img_5.png │ │ │ ├── img_6.png │ │ │ ├── img_7.png │ │ │ └── img_8.png │ │ └── README.md │ ├── deployment │ │ ├── 01-cloud-vendor │ │ │ ├── 01-gcp │ │ │ │ ├── README.md │ │ │ │ ├── img.png │ │ │ │ ├── img_1.png │ │ │ │ ├── img_10.png │ │ │ │ ├── img_11.png │ │ │ │ ├── img_12.png │ │ │ │ ├── img_13.png │ │ │ │ ├── img_14.png │ │ │ │ ├── img_15.png │ │ │ │ ├── img_16.png │ │ │ │ ├── img_17.png │ │ │ │ ├── img_18.png │ │ │ │ ├── img_19.png │ │ │ │ ├── img_2.png │ │ │ │ ├── img_20.png │ │ │ │ ├── img_3.png │ │ │ │ ├── img_4.png │ │ │ │ ├── img_5.png │ │ │ │ ├── img_6.png │ │ │ │ ├── img_7.png │ │ │ │ ├── img_8.png │ │ │ │ └── img_9.png │ │ │ ├── 02-aws │ │ │ │ └── README.md │ │ │ └── README.md │ │ ├── 02-native │ │ │ ├── README.md │ │ │ ├── img.png │ │ │ ├── img_1.png │ │ │ └── img_2.png │ │ ├── 03-docker │ │ │ ├── README.md │ │ │ ├── img.png │ │ │ ├── img_1.png │ │ │ └── img_2.png │ │ ├── 04-docker-compose │ │ │ ├── README.md │ │ │ ├── img.png │ │ │ ├── img_1.png │ │ │ └── img_2.png │ │ ├── 05-kubernetes │ │ │ ├── README.md │ │ │ ├── img.png │ │ │ └── img_1.png │ │ └── README.md │ ├── faq │ │ └── README.md │ ├── getting-started │ │ ├── 01-installation │ │ │ └── README.md │ │ ├── 02-deployment │ │ │ ├── README.md │ │ │ ├── image-20220109204105305.png │ │ │ ├── image-20220109204147240.png │ │ │ ├── image-20220109204211383.png │ │ │ ├── image-20220109204654304.png │ │ │ └── image-20220109204736758.png │ │ ├── 03-writing-a-dag │ │ │ ├── README.md │ │ │ └── image-20220110232821353.png │ │ ├── 04-running-a-dag │ │ │ ├── README.md │ │ │ ├── image-20220111233617011.png │ │ │ ├── image-20220111234022226.png │ │ │ ├── image-20220112000007675.png │ │ │ ├── image-20220112001053445.png │ │ │ ├── image-20220112001907336.png │ │ │ ├── image-20220112002034847.png │ │ │ ├── image-20220112002145920.png │ │ │ ├── image-20220112002706381.png │ │ │ ├── image-20220112002811494.png │ │ │ ├── image-20220112003422433.png │ │ │ ├── image-20220112003515590.png │ │ │ ├── image-20220112003918763.png │ │ │ ├── image-20220112004005070.png │ │ │ ├── image-20220112004132225.png │ │ │ ├── image-20220112005139504.png │ │ │ └── image-20220112005238029.png │ │ ├── 05-outro │ │ │ └── README.md │ │ └── README.md │ ├── index.md │ └── operation │ │ ├── 01-timezone │ │ └── README.md │ │ ├── 02-schedule-interval │ │ └── README.md │ │ ├── 03-cluster-policies │ │ └── README.md │ │ ├── 04-priority-weights │ │ └── README.md │ │ ├── 05-rbac │ │ └── README.md │ │ ├── 06-interfaces │ │ └── README.md │ │ └── README.md └── yarn.lock └── my-airflow-project ├── .gitignore ├── .pre-commit-config.yaml ├── Makefile ├── README.md ├── airflow.cfg ├── dags ├── 01_writing_various_task_flows │ ├── 01_simple_tasks.py │ ├── 02_parallel_tasks.py │ ├── 03_parallel_task_list.py │ ├── 04_conditional_tasks_with_branch.py │ ├── 05_conditional_tasks_with_depends_on_past.py │ ├── 06_conditional_tasks_with_wait_for_downstream.py │ ├── 07_conditional_tasks_with_trigger_rule.py │ └── 08_subdag.py ├── 02_dependencies_between_dags │ ├── 00_to_be_triggered_by_other_dags.py │ ├── 01_trigger_dagrun.py │ └── 02_waiting_tasks_in_another_dagrun.py ├── 03_visualization_and_documentation_dags │ ├── 01_task_groups.py │ ├── 02_labels.py │ └── 03_documentation.py ├── 04_using_various_operators │ ├── 01_bash_operator.py │ ├── 02_python_operator.py │ ├── 03_python_virtualenv_operator.py │ ├── 04_dummy_operator.py │ ├── 05_email_operator.py │ ├── 06_custom_operator.py │ └── 06_custom_operator.sql ├── 05_etc_features │ ├── 01_xcoms.py │ ├── 02_variables.py │ ├── 03_connections_and_hooks.py │ └── 04_templates.py ├── 06_taskflow_api │ ├── 01_concept.py │ ├── 01_concept_pyoperator_version.py │ ├── 02_custom_dependencies.py │ ├── 03_with_operators.py │ ├── 04_virtualenv.py │ └── 05_docker.py └── hello_world.py ├── docker-compose.yml └── requirements.txt /.github/workflows/build_and_deploy.yaml: -------------------------------------------------------------------------------- 1 | # ref: https://milooy.wordpress.com/2020/07/28/github-actions%EB%A1%9C-vuepress-%EB%B0%B0%ED%8F%AC%ED%95%98%EA%B8%B0/ 2 | 3 | name: Build and Deploy 4 | on: 5 | push: 6 | branches: 7 | - main 8 | jobs: 9 | build-and-deploy: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@main 14 | - name: Vuepress deploy 15 | uses: jenkey2011/vuepress-deploy@master 16 | env: 17 | ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} 18 | BUILD_SCRIPT: cd docs && yarn && yarn build 19 | TARGET_BRANCH: gh-pages 20 | BUILD_DIR: src/.vuepress/dist 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | 3 | .idea/ 4 | .vscode/ 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # apache-airflow-tutorials-for-beginner 2 | 3 | Apache Airflow 입문자를 위한 튜토리얼 문서입니다. 4 | 5 | -------------------------------------------------------------------------------- /chart/airflow/.helmignore: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Patterns to ignore when building packages. 19 | # This supports shell glob matching, relative path matching, and 20 | # negation (prefixed with !). Only one pattern per line. 21 | .DS_Store 22 | # Common VCS dirs 23 | .git/ 24 | .gitignore 25 | .bzr/ 26 | .bzrignore 27 | .hg/ 28 | .hgignore 29 | .svn/ 30 | # Common backup files 31 | *.swp 32 | *.bak 33 | *.tmp 34 | *~ 35 | # Various IDEs 36 | .project 37 | .idea/ 38 | *.tmproj 39 | bin 40 | 41 | # We do not want to include our Python Helm Chart Unit test files 42 | tests 43 | -------------------------------------------------------------------------------- /chart/airflow/Chart.lock: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: postgresql 3 | repository: https://charts.bitnami.com/bitnami 4 | version: 10.5.3 5 | digest: sha256:d8ba564b767cbf73a4ca87cb3b97e0a75bc813ba0a58a1b0bd6c7154a608e783 6 | generated: "2021-07-20T23:05:18.37915+01:00" 7 | -------------------------------------------------------------------------------- /chart/airflow/INSTALL: -------------------------------------------------------------------------------- 1 | ## INSTALL / BUILD instructions for Apache Airflow Chart 2 | 3 | # The Assumption here is that you have a running Kubernetes cluster 4 | # and helm installed & configured to talk with the cluster 5 | 6 | # Run `helm dependency update` Command to install dependencies (postgresql) 7 | helm dependency update 8 | 9 | # Run `helm install` Command 10 | helm install airflow . 11 | 12 | # If you want to install in a particular namespace 13 | ## Create that namespace (example 'airflow' here, change it as needed) 14 | kubectl create namespace airflow 15 | 16 | ## Install the chart in that namespace 17 | helm install airflow -n airflow . 18 | -------------------------------------------------------------------------------- /chart/airflow/NOTICE: -------------------------------------------------------------------------------- 1 | Apache Airflow 2 | Copyright 2016-2021 The Apache Software Foundation 3 | 4 | This product includes software developed at The Apache Software 5 | Foundation (http://www.apache.org/). 6 | ======================================================================= 7 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/Chart.lock: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: common 3 | repository: https://charts.bitnami.com/bitnami 4 | version: 1.7.0 5 | digest: sha256:905ccb15952f5bfe132dd73e85c1d984a7c3cc0fc089cd6c272de42093327284 6 | generated: "2021-07-06T00:45:47.873160401Z" 7 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/Chart.yaml: -------------------------------------------------------------------------------- 1 | annotations: 2 | category: Database 3 | apiVersion: v2 4 | appVersion: 11.12.0 5 | dependencies: 6 | - name: common 7 | repository: https://charts.bitnami.com/bitnami 8 | version: 1.x.x 9 | description: Chart for PostgreSQL, an object-relational database management system 10 | (ORDBMS) with an emphasis on extensibility and on standards-compliance. 11 | home: https://github.com/bitnami/charts/tree/master/bitnami/postgresql 12 | icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png 13 | keywords: 14 | - postgresql 15 | - postgres 16 | - database 17 | - sql 18 | - replication 19 | - cluster 20 | maintainers: 21 | - email: containers@bitnami.com 22 | name: Bitnami 23 | - email: cedric@desaintmartin.fr 24 | name: desaintmartin 25 | name: postgresql 26 | sources: 27 | - https://github.com/bitnami/bitnami-docker-postgresql 28 | - https://www.postgresql.org/ 29 | version: 10.5.3 30 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/charts/common/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/charts/common/Chart.yaml: -------------------------------------------------------------------------------- 1 | annotations: 2 | category: Infrastructure 3 | apiVersion: v2 4 | appVersion: 1.7.0 5 | description: A Library Helm Chart for grouping common logic between bitnami charts. 6 | This chart is not deployable by itself. 7 | home: https://github.com/bitnami/charts/tree/master/bitnami/common 8 | icon: https://bitnami.com/downloads/logos/bitnami-mark.png 9 | keywords: 10 | - common 11 | - helper 12 | - template 13 | - function 14 | - bitnami 15 | maintainers: 16 | - email: containers@bitnami.com 17 | name: Bitnami 18 | name: common 19 | sources: 20 | - https://github.com/bitnami/charts 21 | - http://www.bitnami.com/ 22 | type: library 23 | version: 1.7.0 24 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/charts/common/templates/_labels.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Kubernetes standard labels 4 | */}} 5 | {{- define "common.labels.standard" -}} 6 | app.kubernetes.io/name: {{ include "common.names.name" . }} 7 | helm.sh/chart: {{ include "common.names.chart" . }} 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | {{- end -}} 11 | 12 | {{/* 13 | Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector 14 | */}} 15 | {{- define "common.labels.matchLabels" -}} 16 | app.kubernetes.io/name: {{ include "common.names.name" . }} 17 | app.kubernetes.io/instance: {{ .Release.Name }} 18 | {{- end -}} 19 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/charts/common/templates/_names.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "common.names.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create chart name and version as used by the chart label. 11 | */}} 12 | {{- define "common.names.chart" -}} 13 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 14 | {{- end -}} 15 | 16 | {{/* 17 | Create a default fully qualified app name. 18 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 19 | If release name contains chart name it will be used as a full name. 20 | */}} 21 | {{- define "common.names.fullname" -}} 22 | {{- if .Values.fullnameOverride -}} 23 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 24 | {{- else -}} 25 | {{- $name := default .Chart.Name .Values.nameOverride -}} 26 | {{- if contains $name .Release.Name -}} 27 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 28 | {{- else -}} 29 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 30 | {{- end -}} 31 | {{- end -}} 32 | {{- end -}} 33 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/charts/common/templates/_storage.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Return the proper Storage Class 4 | {{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} 5 | */}} 6 | {{- define "common.storage.class" -}} 7 | 8 | {{- $storageClass := .persistence.storageClass -}} 9 | {{- if .global -}} 10 | {{- if .global.storageClass -}} 11 | {{- $storageClass = .global.storageClass -}} 12 | {{- end -}} 13 | {{- end -}} 14 | 15 | {{- if $storageClass -}} 16 | {{- if (eq "-" $storageClass) -}} 17 | {{- printf "storageClassName: \"\"" -}} 18 | {{- else }} 19 | {{- printf "storageClassName: %s" $storageClass -}} 20 | {{- end -}} 21 | {{- end -}} 22 | 23 | {{- end -}} 24 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/charts/common/templates/_tplvalues.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Renders a value that contains template. 4 | Usage: 5 | {{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} 6 | */}} 7 | {{- define "common.tplvalues.render" -}} 8 | {{- if typeIs "string" .value }} 9 | {{- tpl .value .context }} 10 | {{- else }} 11 | {{- tpl (.value | toYaml) .context }} 12 | {{- end }} 13 | {{- end -}} 14 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/charts/common/templates/_warnings.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Warning about using rolling tag. 4 | Usage: 5 | {{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} 6 | */}} 7 | {{- define "common.warnings.rollingTag" -}} 8 | 9 | {{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} 10 | WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. 11 | +info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ 12 | {{- end }} 13 | 14 | {{- end -}} 15 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/charts/common/values.yaml: -------------------------------------------------------------------------------- 1 | ## bitnami/common 2 | ## It is required by CI/CD tools and processes. 3 | exampleValue: common-chart 4 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/ci/commonAnnotations.yaml: -------------------------------------------------------------------------------- 1 | commonAnnotations: 2 | helm.sh/hook: "\"pre-install, pre-upgrade\"" 3 | helm.sh/hook-weight: "-1" 4 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/ci/default-values.yaml: -------------------------------------------------------------------------------- 1 | # Leave this file empty to ensure that CI runs builds against the default configuration in values.yaml. 2 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/ci/shmvolume-disabled-values.yaml: -------------------------------------------------------------------------------- 1 | shmVolume: 2 | enabled: false 3 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/files/README.md: -------------------------------------------------------------------------------- 1 | Copy here your postgresql.conf and/or pg_hba.conf files to use it as a config map. 2 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/files/conf.d/README.md: -------------------------------------------------------------------------------- 1 | If you don't want to provide the whole configuration file and only specify certain parameters, you can copy here your extended `.conf` files. 2 | These files will be injected as a config maps and add/overwrite the default configuration using the `include_dir` directive that allows settings to be loaded from files other than the default `postgresql.conf`. 3 | 4 | More info in the [bitnami-docker-postgresql README](https://github.com/bitnami/bitnami-docker-postgresql#configuration-file). 5 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/files/docker-entrypoint-initdb.d/README.md: -------------------------------------------------------------------------------- 1 | You can copy here your custom `.sh`, `.sql` or `.sql.gz` file so they are executed during the first boot of the image. 2 | 3 | More info in the [bitnami-docker-postgresql](https://github.com/bitnami/bitnami-docker-postgresql#initializing-a-new-instance) repository. 4 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | {{ if and (or (.Files.Glob "files/postgresql.conf") (.Files.Glob "files/pg_hba.conf") .Values.postgresqlConfiguration .Values.pgHbaConfiguration) (not .Values.configurationConfigMap) }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ template "common.names.fullname" . }}-configuration 6 | labels: 7 | {{- include "common.labels.standard" . | nindent 4 }} 8 | {{- if .Values.commonAnnotations }} 9 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 10 | {{- end }} 11 | namespace: {{ .Release.Namespace }} 12 | data: 13 | {{- if (.Files.Glob "files/postgresql.conf") }} 14 | {{ (.Files.Glob "files/postgresql.conf").AsConfig | indent 2 }} 15 | {{- else if .Values.postgresqlConfiguration }} 16 | postgresql.conf: | 17 | {{- range $key, $value := default dict .Values.postgresqlConfiguration }} 18 | {{- if kindIs "string" $value }} 19 | {{ $key | snakecase }} = '{{ $value }}' 20 | {{- else }} 21 | {{ $key | snakecase }} = {{ $value }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | {{- if (.Files.Glob "files/pg_hba.conf") }} 26 | {{ (.Files.Glob "files/pg_hba.conf").AsConfig | indent 2 }} 27 | {{- else if .Values.pgHbaConfiguration }} 28 | pg_hba.conf: | 29 | {{ .Values.pgHbaConfiguration | indent 4 }} 30 | {{- end }} 31 | {{ end }} 32 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/extended-config-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (or (.Files.Glob "files/conf.d/*.conf") .Values.postgresqlExtendedConf) (not .Values.extendedConfConfigMap)}} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ template "common.names.fullname" . }}-extended-configuration 6 | labels: 7 | {{- include "common.labels.standard" . | nindent 4 }} 8 | {{- if .Values.commonAnnotations }} 9 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 10 | {{- end }} 11 | namespace: {{ .Release.Namespace }} 12 | data: 13 | {{- with .Files.Glob "files/conf.d/*.conf" }} 14 | {{ .AsConfig | indent 2 }} 15 | {{- end }} 16 | {{ with .Values.postgresqlExtendedConf }} 17 | override.conf: | 18 | {{- range $key, $value := . }} 19 | {{- if kindIs "string" $value }} 20 | {{ $key | snakecase }} = '{{ $value }}' 21 | {{- else }} 22 | {{ $key | snakecase }} = {{ $value }} 23 | {{- end }} 24 | {{- end }} 25 | {{- end }} 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/extra-list.yaml: -------------------------------------------------------------------------------- 1 | {{- range .Values.extraDeploy }} 2 | --- 3 | {{ include "common.tplvalues.render" (dict "value" . "context" $) }} 4 | {{- end }} 5 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/initialization-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (or (.Files.Glob "files/docker-entrypoint-initdb.d/*.{sh,sql,sql.gz}") .Values.initdbScripts) (not .Values.initdbScriptsConfigMap) }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ template "common.names.fullname" . }}-init-scripts 6 | labels: 7 | {{- include "common.labels.standard" . | nindent 4 }} 8 | {{- if .Values.commonAnnotations }} 9 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 10 | {{- end }} 11 | namespace: {{ .Release.Namespace }} 12 | {{- with .Files.Glob "files/docker-entrypoint-initdb.d/*.sql.gz" }} 13 | binaryData: 14 | {{- range $path, $bytes := . }} 15 | {{ base $path }}: {{ $.Files.Get $path | b64enc | quote }} 16 | {{- end }} 17 | {{- end }} 18 | data: 19 | {{- with .Files.Glob "files/docker-entrypoint-initdb.d/*.{sh,sql}" }} 20 | {{ .AsConfig | indent 2 }} 21 | {{- end }} 22 | {{- with .Values.initdbScripts }} 23 | {{ toYaml . | indent 2 }} 24 | {{- end }} 25 | {{- end }} 26 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/metrics-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.metrics.enabled .Values.metrics.customMetrics }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ template "postgresql.metricsCM" . }} 6 | labels: 7 | {{- include "common.labels.standard" . | nindent 4 }} 8 | {{- if .Values.commonAnnotations }} 9 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 10 | {{- end }} 11 | namespace: {{ .Release.Namespace }} 12 | data: 13 | custom-metrics.yaml: {{ toYaml .Values.metrics.customMetrics | quote }} 14 | {{- end }} 15 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/metrics-svc.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.metrics.enabled }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ template "common.names.fullname" . }}-metrics 6 | labels: 7 | {{- include "common.labels.standard" . | nindent 4 }} 8 | annotations: 9 | {{- if .Values.commonAnnotations }} 10 | {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 11 | {{- end }} 12 | {{- toYaml .Values.metrics.service.annotations | nindent 4 }} 13 | namespace: {{ .Release.Namespace }} 14 | spec: 15 | type: {{ .Values.metrics.service.type }} 16 | {{- if and (eq .Values.metrics.service.type "LoadBalancer") .Values.metrics.service.loadBalancerIP }} 17 | loadBalancerIP: {{ .Values.metrics.service.loadBalancerIP }} 18 | {{- end }} 19 | ports: 20 | - name: http-metrics 21 | port: 9187 22 | targetPort: http-metrics 23 | selector: 24 | {{- include "common.labels.matchLabels" . | nindent 4 }} 25 | role: primary 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/networkpolicy.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.networkPolicy.enabled }} 2 | kind: NetworkPolicy 3 | apiVersion: {{ template "postgresql.networkPolicy.apiVersion" . }} 4 | metadata: 5 | name: {{ template "common.names.fullname" . }} 6 | labels: 7 | {{- include "common.labels.standard" . | nindent 4 }} 8 | {{- if .Values.commonAnnotations }} 9 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 10 | {{- end }} 11 | namespace: {{ .Release.Namespace }} 12 | spec: 13 | podSelector: 14 | matchLabels: 15 | {{- include "common.labels.matchLabels" . | nindent 6 }} 16 | ingress: 17 | # Allow inbound connections 18 | - ports: 19 | - port: {{ template "postgresql.port" . }} 20 | {{- if not .Values.networkPolicy.allowExternal }} 21 | from: 22 | - podSelector: 23 | matchLabels: 24 | {{ template "common.names.fullname" . }}-client: "true" 25 | {{- if .Values.networkPolicy.explicitNamespacesSelector }} 26 | namespaceSelector: 27 | {{ toYaml .Values.networkPolicy.explicitNamespacesSelector | indent 12 }} 28 | {{- end }} 29 | - podSelector: 30 | matchLabels: 31 | {{- include "common.labels.matchLabels" . | nindent 14 }} 32 | role: read 33 | {{- end }} 34 | {{- if .Values.metrics.enabled }} 35 | # Allow prometheus scrapes 36 | - ports: 37 | - port: 9187 38 | {{- end }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/podsecuritypolicy.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.psp.create }} 2 | apiVersion: {{ include "podsecuritypolicy.apiVersion" . }} 3 | kind: PodSecurityPolicy 4 | metadata: 5 | name: {{ template "common.names.fullname" . }} 6 | labels: 7 | {{- include "common.labels.standard" . | nindent 4 }} 8 | {{- if .Values.commonAnnotations }} 9 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 10 | {{- end }} 11 | namespace: {{ .Release.Namespace }} 12 | spec: 13 | privileged: false 14 | volumes: 15 | - 'configMap' 16 | - 'secret' 17 | - 'persistentVolumeClaim' 18 | - 'emptyDir' 19 | - 'projected' 20 | hostNetwork: false 21 | hostIPC: false 22 | hostPID: false 23 | runAsUser: 24 | rule: 'RunAsAny' 25 | seLinux: 26 | rule: 'RunAsAny' 27 | supplementalGroups: 28 | rule: 'MustRunAs' 29 | ranges: 30 | - min: 1 31 | max: 65535 32 | fsGroup: 33 | rule: 'MustRunAs' 34 | ranges: 35 | - min: 1 36 | max: 65535 37 | readOnlyRootFilesystem: false 38 | {{- end }} 39 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/prometheusrule.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: PrometheusRule 4 | metadata: 5 | name: {{ template "common.names.fullname" . }} 6 | {{- with .Values.metrics.prometheusRule.namespace }} 7 | namespace: {{ . }} 8 | {{- end }} 9 | labels: 10 | {{- include "common.labels.standard" . | nindent 4 }} 11 | {{- with .Values.metrics.prometheusRule.additionalLabels }} 12 | {{- toYaml . | nindent 4 }} 13 | {{- end }} 14 | {{- if .Values.commonAnnotations }} 15 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | {{- with .Values.metrics.prometheusRule.rules }} 19 | groups: 20 | - name: {{ template "postgresql.name" $ }} 21 | rules: {{ tpl (toYaml .) $ | nindent 8 }} 22 | {{- end }} 23 | {{- end }} 24 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/role.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create }} 2 | kind: Role 3 | apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} 4 | metadata: 5 | name: {{ template "common.names.fullname" . }} 6 | labels: 7 | {{- include "common.labels.standard" . | nindent 4 }} 8 | {{- if .Values.commonAnnotations }} 9 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 10 | {{- end }} 11 | namespace: {{ .Release.Namespace }} 12 | rules: 13 | {{- if .Values.psp.create }} 14 | - apiGroups: ["extensions"] 15 | resources: ["podsecuritypolicies"] 16 | verbs: ["use"] 17 | resourceNames: 18 | - {{ template "common.names.fullname" . }} 19 | {{- end }} 20 | {{- end }} 21 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create }} 2 | kind: RoleBinding 3 | apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} 4 | metadata: 5 | name: {{ template "common.names.fullname" . }} 6 | labels: 7 | {{- include "common.labels.standard" . | nindent 4 }} 8 | {{- if .Values.commonAnnotations }} 9 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 10 | {{- end }} 11 | namespace: {{ .Release.Namespace }} 12 | roleRef: 13 | kind: Role 14 | name: {{ template "common.names.fullname" . }} 15 | apiGroup: rbac.authorization.k8s.io 16 | subjects: 17 | - kind: ServiceAccount 18 | name: {{ default (include "common.names.fullname" . ) .Values.serviceAccount.name }} 19 | namespace: {{ .Release.Namespace }} 20 | {{- end }} 21 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- if (include "postgresql.createSecret" .) }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ template "common.names.fullname" . }} 6 | labels: 7 | {{- include "common.labels.standard" . | nindent 4 }} 8 | {{- if .Values.commonAnnotations }} 9 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 10 | {{- end }} 11 | namespace: {{ .Release.Namespace }} 12 | type: Opaque 13 | data: 14 | {{- if not (eq (include "postgresql.username" .) "postgres") }} 15 | postgresql-postgres-password: {{ include "postgresql.postgres.password" . | b64enc | quote }} 16 | {{- end }} 17 | postgresql-password: {{ include "postgresql.password" . | b64enc | quote }} 18 | {{- if .Values.replication.enabled }} 19 | postgresql-replication-password: {{ include "postgresql.replication.password" . | b64enc | quote }} 20 | {{- end }} 21 | {{- if (and .Values.ldap.enabled .Values.ldap.bind_password)}} 22 | postgresql-ldap-password: {{ .Values.ldap.bind_password | b64enc | quote }} 23 | {{- end }} 24 | {{- end -}} 25 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (.Values.serviceAccount.enabled) (not .Values.serviceAccount.name) }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | {{- include "common.labels.standard" . | nindent 4 }} 7 | name: {{ template "common.names.fullname" . }} 8 | {{- if .Values.commonAnnotations }} 9 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 10 | {{- end }} 11 | namespace: {{ .Release.Namespace }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | name: {{ include "common.names.fullname" . }} 6 | {{- if .Values.metrics.serviceMonitor.namespace }} 7 | namespace: {{ .Values.metrics.serviceMonitor.namespace }} 8 | {{- end }} 9 | labels: 10 | {{- include "common.labels.standard" . | nindent 4 }} 11 | {{- if .Values.metrics.serviceMonitor.additionalLabels }} 12 | {{- toYaml .Values.metrics.serviceMonitor.additionalLabels | nindent 4 }} 13 | {{- end }} 14 | {{- if .Values.commonAnnotations }} 15 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 16 | {{- end }} 17 | 18 | spec: 19 | endpoints: 20 | - port: http-metrics 21 | {{- if .Values.metrics.serviceMonitor.interval }} 22 | interval: {{ .Values.metrics.serviceMonitor.interval }} 23 | {{- end }} 24 | {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} 25 | scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} 26 | {{- end }} 27 | namespaceSelector: 28 | matchNames: 29 | - {{ .Release.Namespace }} 30 | selector: 31 | matchLabels: 32 | {{- include "common.labels.matchLabels" . | nindent 6 }} 33 | {{- end }} 34 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/svc-headless.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ template "common.names.fullname" . }}-headless 5 | labels: 6 | {{- include "common.labels.standard" . | nindent 4 }} 7 | {{- if .Values.commonAnnotations }} 8 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 9 | {{- end }} 10 | # Use this annotation in addition to the actual publishNotReadyAddresses 11 | # field below because the annotation will stop being respected soon but the 12 | # field is broken in some versions of Kubernetes: 13 | # https://github.com/kubernetes/kubernetes/issues/58662 14 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 15 | namespace: {{ .Release.Namespace }} 16 | spec: 17 | type: ClusterIP 18 | clusterIP: None 19 | # We want all pods in the StatefulSet to have their addresses published for 20 | # the sake of the other Postgresql pods even before they're ready, since they 21 | # have to be able to talk to each other in order to become ready. 22 | publishNotReadyAddresses: true 23 | ports: 24 | - name: tcp-postgresql 25 | port: {{ template "postgresql.port" . }} 26 | targetPort: tcp-postgresql 27 | selector: 28 | {{- include "common.labels.matchLabels" . | nindent 4 }} 29 | -------------------------------------------------------------------------------- /chart/airflow/charts/postgresql/templates/tls-secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- if (include "postgresql.createTlsSecret" . )}} 2 | {{- $ca := genCA "postgresql-ca" 365 }} 3 | {{- $fullname := include "common.names.fullname" . }} 4 | {{- $releaseNamespace := .Release.Namespace }} 5 | {{- $clusterDomain := .Values.clusterDomain }} 6 | {{- $headlessServiceName := printf "%s-headless" (include "common.names.fullname" .) }} 7 | {{- $altNames := list (printf "*.%s.%s.svc.%s" $fullname $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $fullname $releaseNamespace $clusterDomain) (printf "*.%s.%s.svc.%s" $headlessServiceName $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $headlessServiceName $releaseNamespace $clusterDomain) $fullname }} 8 | {{- $crt := genSignedCert $fullname nil $altNames 365 $ca }} 9 | apiVersion: v1 10 | kind: Secret 11 | metadata: 12 | name: {{ printf "%s-crt" (include "common.names.fullname" .) }} 13 | labels: {{- include "common.labels.standard" . | nindent 4 }} 14 | {{- if .Values.commonLabels }} 15 | {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} 16 | {{- end }} 17 | {{- if .Values.commonAnnotations }} 18 | annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} 19 | {{- end }} 20 | type: kubernetes.io/tls 21 | data: 22 | ca.crt: {{ $ca.Cert | b64enc | quote }} 23 | tls.crt: {{ $crt.Cert | b64enc | quote }} 24 | tls.key: {{ $crt.Key | b64enc | quote }} 25 | {{- end }} 26 | -------------------------------------------------------------------------------- /chart/airflow/dockerfiles/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | Those are images that are needed for the Helm Chart. 21 | 22 | In each of the images you can find "build_and_push.sh" script that builds and pushes the image. 23 | 24 | You need to be a PMC with direct push access to "apache/airflow" DockerHub registry 25 | to be able to push to the Airflow DockerHub registry. 26 | 27 | You can set the DOCKERHUB_USER variable to push to your own DockerHub user if you want 28 | to test the image or build your own image. 29 | -------------------------------------------------------------------------------- /chart/airflow/dockerfiles/statsd-exporter/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | ARG STATSD_VERSION="missing_version" 18 | 19 | FROM prom/statsd-exporter:${STATSD_VERSION} 20 | 21 | ARG STATSD_VERSION 22 | ARG AIRFLOW_STATSD_EXPORTER_VERSION 23 | ARG COMMIT_SHA 24 | 25 | LABEL org.apache.airflow.component="statsd-exporter" \ 26 | org.apache.airflow.stasd.version="${STATSD_VERSION}" \ 27 | org.apache.airflow.airflow-stasd-exporter.version="${AIRFLOW_STATSD_EXPORTER_VERSION}" \ 28 | org.apache.airflow.commit-sha="${COMMIT_SHA}" \ 29 | maintainer="Apache Airflow Community " 30 | 31 | COPY mappings.yml /etc/statsd-exporter/mappings.yml 32 | -------------------------------------------------------------------------------- /chart/airflow/templates/cleanup/cleanup-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow Cleanup ServiceAccount 20 | ################################# 21 | {{- if and .Values.cleanup.serviceAccount.create .Values.cleanup.enabled }} 22 | kind: ServiceAccount 23 | apiVersion: v1 24 | metadata: 25 | name: {{ include "cleanup.serviceAccountName" . }} 26 | labels: 27 | tier: airflow 28 | release: {{ .Release.Name }} 29 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 30 | heritage: {{ .Release.Service }} 31 | {{- with .Values.labels }} 32 | {{ toYaml . | nindent 4 }} 33 | {{- end }} 34 | {{- with .Values.cleanup.serviceAccount.annotations }} 35 | annotations: 36 | {{ toYaml . | nindent 4 }} 37 | {{- end }} 38 | {{- end }} 39 | -------------------------------------------------------------------------------- /chart/airflow/templates/configmaps/statsd-configmap.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow StatsD ConfigMap 20 | ################################# 21 | {{- if and .Values.statsd.enabled .Values.statsd.extraMappings }} 22 | apiVersion: v1 23 | kind: ConfigMap 24 | metadata: 25 | name: {{ .Release.Name }}-statsd 26 | labels: 27 | tier: airflow 28 | component: config 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | indent 4 }} 34 | {{- end }} 35 | data: 36 | mappings.yml: |- 37 | {{ .Files.Get "dockerfiles/statsd-exporter/mappings.yml" | indent 4 }} 38 | {{ toYaml .Values.statsd.extraMappings | indent 6 }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/templates/configmaps/webserver-configmap.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow ConfigMap 20 | ################################# 21 | {{- if .Values.webserver.webserverConfig }} 22 | apiVersion: v1 23 | kind: ConfigMap 24 | metadata: 25 | name: {{ .Release.Name }}-webserver-config 26 | labels: 27 | tier: airflow 28 | component: config 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | indent 4 }} 34 | {{- end }} 35 | data: 36 | webserver_config.py: | 37 | {{- tpl .Values.webserver.webserverConfig . | nindent 4 }} 38 | {{- end }} 39 | -------------------------------------------------------------------------------- /chart/airflow/templates/flower/flower-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ###################################### 19 | ## Airflow Flower ServiceAccount 20 | ###################################### 21 | {{- if and .Values.flower.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) .Values.flower.serviceAccount.create }} 22 | kind: ServiceAccount 23 | apiVersion: v1 24 | metadata: 25 | name: {{ include "flower.serviceAccountName" . }} 26 | labels: 27 | tier: airflow 28 | component: flower 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | nindent 4 }} 34 | {{- end }} 35 | {{- with .Values.flower.serviceAccount.annotations }} 36 | annotations: 37 | {{ toYaml . | nindent 4 }} 38 | {{- end }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/templates/jobs/create-user-job-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ########################################### 19 | ## Airflow Create User Job ServiceAccount 20 | ########################################### 21 | {{- if and .Values.createUserJob.serviceAccount.create .Values.webserver.defaultUser.enabled }} 22 | kind: ServiceAccount 23 | apiVersion: v1 24 | metadata: 25 | name: {{ include "createUserJob.serviceAccountName" . }} 26 | labels: 27 | tier: airflow 28 | component: create-user-job 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | nindent 4 }} 34 | {{- end }} 35 | {{- with .Values.createUserJob.serviceAccount.annotations }} 36 | annotations: 37 | {{ toYaml . | nindent 4 }} 38 | {{- end }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/templates/jobs/migrate-database-job-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ############################################# 19 | ## Airflow Migrate Database Job ServiceAccount 20 | ############################################## 21 | {{- if .Values.migrateDatabaseJob.serviceAccount.create }} 22 | kind: ServiceAccount 23 | apiVersion: v1 24 | metadata: 25 | name: {{ include "migrateDatabaseJob.serviceAccountName" . }} 26 | labels: 27 | tier: airflow 28 | component: run-airflow-migrations 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | nindent 4 }} 34 | {{- end }} 35 | {{- with .Values.migrateDatabaseJob.serviceAccount.annotations }} 36 | annotations: 37 | {{ toYaml . | nindent 4 }} 38 | {{- end }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/templates/limitrange.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow Namespace LimitRange 20 | ################################# 21 | {{- if .Values.limits }} 22 | apiVersion: v1 23 | kind: LimitRange 24 | metadata: 25 | name: {{ .Release.Name }}-limit-range 26 | labels: 27 | tier: resources 28 | component: limitrange 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | indent 4 }} 34 | {{- end }} 35 | spec: 36 | limits: 37 | {{ toYaml .Values.limits | indent 4 }} 38 | {{- end }} 39 | -------------------------------------------------------------------------------- /chart/airflow/templates/pgbouncer/pgbouncer-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ###################################### 19 | ## Airflow Pgbouncer ServiceAccount 20 | ###################################### 21 | {{- if and .Values.pgbouncer.serviceAccount.create .Values.pgbouncer.enabled }} 22 | kind: ServiceAccount 23 | apiVersion: v1 24 | metadata: 25 | name: {{ include "pgbouncer.serviceAccountName" . }} 26 | labels: 27 | tier: airflow 28 | component: pgbouncer 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | nindent 4 }} 34 | {{- end }} 35 | {{- with .Values.pgbouncer.serviceAccount.annotations }} 36 | annotations: 37 | {{ toYaml . | nindent 4 }} 38 | {{- end }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/templates/rbac/pod-cleanup-role.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow Cleanup Role 20 | ################################# 21 | {{- if and .Values.rbac.create .Values.cleanup.enabled }} 22 | kind: Role 23 | apiVersion: rbac.authorization.k8s.io/v1 24 | metadata: 25 | name: {{ .Release.Name }}-cleanup-role 26 | labels: 27 | tier: airflow 28 | release: {{ .Release.Name }} 29 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 30 | heritage: {{ .Release.Service }} 31 | {{- with .Values.labels }} 32 | {{ toYaml . | indent 4 }} 33 | {{- end }} 34 | rules: 35 | - apiGroups: 36 | - "" 37 | resources: 38 | - "pods" 39 | verbs: 40 | - "list" 41 | - "delete" 42 | {{- end }} 43 | -------------------------------------------------------------------------------- /chart/airflow/templates/rbac/pod-cleanup-rolebinding.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow Cleanup Role Binding 20 | ################################# 21 | {{- if and .Values.rbac.create .Values.cleanup.enabled }} 22 | kind: RoleBinding 23 | apiVersion: rbac.authorization.k8s.io/v1 24 | metadata: 25 | name: {{ .Release.Name }}-cleanup-rolebinding 26 | labels: 27 | tier: airflow 28 | release: {{ .Release.Name }} 29 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 30 | heritage: {{ .Release.Service }} 31 | {{- with .Values.labels }} 32 | {{ toYaml . | indent 4 }} 33 | {{- end }} 34 | roleRef: 35 | apiGroup: rbac.authorization.k8s.io 36 | kind: Role 37 | name: {{ .Release.Name }}-cleanup-role 38 | subjects: 39 | - kind: ServiceAccount 40 | name: {{ include "cleanup.serviceAccountName" . }} 41 | namespace: "{{ .Release.Namespace }}" 42 | {{- end }} 43 | -------------------------------------------------------------------------------- /chart/airflow/templates/redis/redis-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ###################################### 19 | ## Airflow Redis ServiceAccount 20 | ###################################### 21 | {{- if and .Values.redis.enabled .Values.redis.serviceAccount.create (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }} 22 | kind: ServiceAccount 23 | apiVersion: v1 24 | metadata: 25 | name: {{ include "redis.serviceAccountName" . }} 26 | labels: 27 | tier: airflow 28 | component: redis 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | nindent 4 }} 34 | {{- end }} 35 | {{- with .Values.redis.serviceAccount.annotations }} 36 | annotations: 37 | {{ toYaml . | nindent 4 }} 38 | {{- end }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/templates/resourcequota.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow Namespace ResourceQuota 20 | ################################# 21 | {{- if .Values.quotas }} 22 | apiVersion: v1 23 | kind: ResourceQuota 24 | metadata: 25 | name: {{ .Release.Name }}-resource-quota 26 | labels: 27 | tier: resources 28 | component: resourcequota 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | indent 4 }} 34 | {{- end }} 35 | spec: 36 | hard: 37 | {{ toYaml .Values.quotas | indent 4 }} 38 | {{- end }} 39 | -------------------------------------------------------------------------------- /chart/airflow/templates/scheduler/scheduler-service.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow Scheduler Service 20 | ################################# 21 | {{- if eq .Values.executor "LocalExecutor" }} 22 | kind: Service 23 | apiVersion: v1 24 | metadata: 25 | name: {{ .Release.Name }}-scheduler 26 | labels: 27 | tier: airflow 28 | component: scheduler 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | indent 4 }} 34 | {{- end }} 35 | spec: 36 | clusterIP: None 37 | selector: 38 | tier: airflow 39 | component: scheduler 40 | release: {{ .Release.Name }} 41 | ports: 42 | - name: task-logs 43 | protocol: TCP 44 | port: {{ .Values.ports.workerLogs }} 45 | targetPort: {{ .Values.ports.workerLogs }} 46 | {{- end }} 47 | -------------------------------------------------------------------------------- /chart/airflow/templates/scheduler/scheduler-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow Scheduler ServiceAccount 20 | ################################# 21 | {{- if .Values.scheduler.serviceAccount.create }} 22 | kind: ServiceAccount 23 | apiVersion: v1 24 | metadata: 25 | name: {{ include "scheduler.serviceAccountName" . }} 26 | labels: 27 | tier: airflow 28 | component: scheduler 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | nindent 4 }} 34 | {{- end }} 35 | {{- with .Values.scheduler.serviceAccount.annotations }} 36 | annotations: 37 | {{- range $key, $value := . }} 38 | {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }} 39 | {{- end }} 40 | {{- end }} 41 | {{- end }} 42 | -------------------------------------------------------------------------------- /chart/airflow/templates/secrets/elasticsearch-secret.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Elasticsearch Secret 20 | ################################# 21 | {{- if (and .Values.elasticsearch.enabled (not .Values.elasticsearch.secretName)) }} 22 | kind: Secret 23 | apiVersion: v1 24 | metadata: 25 | name: {{ .Release.Name }}-elasticsearch 26 | labels: 27 | release: {{ .Release.Name }} 28 | chart: {{ .Chart.Name }} 29 | heritage: {{ .Release.Service }} 30 | {{- with .Values.labels }} 31 | {{ toYaml . | indent 4 }} 32 | {{- end }} 33 | type: Opaque 34 | data: 35 | {{- with .Values.elasticsearch.connection }} 36 | connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery)) "host" (printf "%s:%s" .host ((default 9200 .port) | toString) ) ) | b64enc | quote }} 37 | {{- end }} 38 | {{- end }} 39 | -------------------------------------------------------------------------------- /chart/airflow/templates/secrets/fernetkey-secret.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow Fernet Key Secret 20 | ################################# 21 | {{- if not .Values.fernetKeySecretName }} 22 | {{ $generated_fernet_key := (randAlphaNum 32 | b64enc) }} 23 | kind: Secret 24 | apiVersion: v1 25 | metadata: 26 | name: {{ .Release.Name }}-fernet-key 27 | labels: 28 | tier: airflow 29 | release: {{ .Release.Name }} 30 | chart: {{ .Chart.Name }} 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | indent 4 }} 34 | {{- end }} 35 | annotations: 36 | "helm.sh/hook": "pre-install" 37 | "helm.sh/hook-delete-policy": "before-hook-creation" 38 | "helm.sh/hook-weight": "0" 39 | type: Opaque 40 | data: 41 | fernet-key: {{ (default $generated_fernet_key .Values.fernetKey) | b64enc | quote }} 42 | {{- end }} 43 | -------------------------------------------------------------------------------- /chart/airflow/templates/secrets/flower-secret.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Flower Secret 20 | ################################# 21 | {{- if (and (not .Values.flower.secretName) .Values.flower.username .Values.flower.password) }} 22 | kind: Secret 23 | apiVersion: v1 24 | metadata: 25 | name: {{ .Release.Name }}-flower 26 | labels: 27 | release: {{ .Release.Name }} 28 | chart: {{ .Chart.Name }} 29 | heritage: {{ .Release.Service }} 30 | {{- with .Values.labels }} 31 | {{ toYaml . | indent 4 }} 32 | {{- end }} 33 | type: Opaque 34 | data: 35 | basicAuth: {{ (printf "%s:%s" .Values.flower.username .Values.flower.password) | b64enc | quote }} 36 | {{- end }} 37 | -------------------------------------------------------------------------------- /chart/airflow/templates/secrets/kerberos-keytab-secret.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | {{ if .Values.kerberos.keytabBase64Content }} 19 | apiVersion: v1 20 | metadata: 21 | name: {{ include "kerberos_keytab_secret" . | quote }} 22 | labels: 23 | tier: airflow 24 | component: webserver 25 | release: {{ .Release.Name }} 26 | chart: {{ .Chart.Name }} 27 | heritage: {{ .Release.Service }} 28 | {{- with .Values.labels }} 29 | {{ toYaml . | indent 4 }} 30 | {{- end }} 31 | data: 32 | kerberos.keytab: {{ .Values.kerberos.keytabBase64Content }} 33 | kind: Secret 34 | type: Opaque 35 | {{ end }} 36 | -------------------------------------------------------------------------------- /chart/airflow/templates/secrets/pgbouncer-config-secret.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Pgbouncer Config Secret 20 | ################################# 21 | {{- if (and .Values.pgbouncer.enabled (not .Values.pgbouncer.configSecretName)) }} 22 | kind: Secret 23 | apiVersion: v1 24 | metadata: 25 | name: {{ template "pgbouncer_config_secret" . }} 26 | labels: 27 | tier: airflow 28 | component: pgbouncer 29 | release: {{ .Release.Name }} 30 | chart: {{ .Chart.Name }} 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | indent 4 }} 34 | {{- end }} 35 | type: Opaque 36 | data: 37 | pgbouncer.ini: {{ include "pgbouncer_config" . | b64enc }} 38 | users.txt: {{ include "pgbouncer_users" . | b64enc }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/templates/secrets/registry-secret.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Registry Secret 20 | ################################# 21 | {{- if (and .Values.registry.connection (not .Values.registry.secretName)) }} 22 | kind: Secret 23 | apiVersion: v1 24 | metadata: 25 | name: {{ .Release.Name }}-registry 26 | labels: 27 | release: {{ .Release.Name }} 28 | chart: {{ .Chart.Name }} 29 | heritage: {{ .Release.Service }} 30 | {{- with .Values.labels }} 31 | {{ toYaml . | indent 4 }} 32 | {{- end }} 33 | type: kubernetes.io/dockerconfigjson 34 | data: 35 | .dockerconfigjson: {{ include "registry_docker_config" . | b64enc }} 36 | {{- end }} 37 | -------------------------------------------------------------------------------- /chart/airflow/templates/secrets/webserver-secret-key-secret.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ############################################ 19 | ## Airflow Webserver Flask Secret Key Secret 20 | ############################################ 21 | {{- if not .Values.webserverSecretKeySecretName }} 22 | {{ $generated_secret_key := (randAlphaNum 32 | b64enc) }} 23 | kind: Secret 24 | apiVersion: v1 25 | metadata: 26 | name: {{ .Release.Name }}-webserver-secret-key 27 | labels: 28 | tier: airflow 29 | component: webserver 30 | release: {{ .Release.Name }} 31 | chart: {{ .Chart.Name }} 32 | heritage: {{ .Release.Service }} 33 | {{- with .Values.labels }} 34 | {{ toYaml . | indent 4 }} 35 | {{- end }} 36 | type: Opaque 37 | data: 38 | webserver-secret-key: {{ (default $generated_secret_key .Values.webserverSecretKey) | b64enc | quote }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/templates/statsd/statsd-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ###################################### 19 | ## Airflow StatsD ServiceAccount 20 | ###################################### 21 | {{- if and .Values.statsd.enabled .Values.statsd.serviceAccount.create }} 22 | kind: ServiceAccount 23 | apiVersion: v1 24 | metadata: 25 | name: {{ include "statsd.serviceAccountName" . }} 26 | labels: 27 | tier: airflow 28 | component: statsd 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | nindent 4 }} 34 | {{- end }} 35 | {{- with .Values.statsd.serviceAccount.annotations }} 36 | annotations: 37 | {{ toYaml . | nindent 4 }} 38 | {{- end }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/templates/triggerer/triggerer-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow Triggerer ServiceAccount 20 | ################################# 21 | {{- if semverCompare ">=2.2.0" .Values.airflowVersion }} 22 | {{- if and .Values.triggerer.serviceAccount.create }} 23 | kind: ServiceAccount 24 | apiVersion: v1 25 | metadata: 26 | name: {{ include "triggerer.serviceAccountName" . }} 27 | labels: 28 | tier: airflow 29 | component: triggerer 30 | release: {{ .Release.Name }} 31 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 32 | heritage: {{ .Release.Service }} 33 | {{- with .Values.labels }} 34 | {{ toYaml . | nindent 4 }} 35 | {{- end }} 36 | {{- with .Values.triggerer.serviceAccount.annotations}} 37 | annotations: 38 | {{ toYaml . | nindent 4 }} 39 | {{- end }} 40 | {{- end }} 41 | {{- end }} 42 | -------------------------------------------------------------------------------- /chart/airflow/templates/webserver/webserver-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ###################################### 19 | ## Airflow Webserver ServiceAccount 20 | ###################################### 21 | {{- if .Values.webserver.serviceAccount.create }} 22 | kind: ServiceAccount 23 | apiVersion: v1 24 | metadata: 25 | name: {{ include "webserver.serviceAccountName" . }} 26 | labels: 27 | tier: airflow 28 | component: webserver 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | nindent 4 }} 34 | {{- end }} 35 | {{- with .Values.webserver.serviceAccount.annotations }} 36 | annotations: 37 | {{ toYaml . | nindent 4 }} 38 | {{- end }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /chart/airflow/templates/workers/worker-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ################################ 19 | ## Airflow Worker ServiceAccount 20 | ################################# 21 | {{- if and .Values.workers.serviceAccount.create (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") (eq .Values.executor "KubernetesExecutor")) }} 22 | kind: ServiceAccount 23 | apiVersion: v1 24 | metadata: 25 | name: {{ include "worker.serviceAccountName" . }} 26 | labels: 27 | tier: airflow 28 | component: worker 29 | release: {{ .Release.Name }} 30 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 31 | heritage: {{ .Release.Service }} 32 | {{- with .Values.labels }} 33 | {{ toYaml . | nindent 4 }} 34 | {{- end }} 35 | {{- with .Values.workers.serviceAccount.annotations}} 36 | annotations: 37 | {{ toYaml . | nindent 4 }} 38 | {{- end }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /docs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "airflow-tutorials", 3 | "version": "0.0.1", 4 | "description": "airflow tutorials for beginner", 5 | "main": "index.js", 6 | "authors": { 7 | "name": "heumsi", 8 | "email": "heumsi@naver.com" 9 | }, 10 | "repository": "/airflow-tutorials", 11 | "scripts": { 12 | "dev": "vuepress dev src", 13 | "build": "vuepress build src" 14 | }, 15 | "license": "MIT", 16 | "devDependencies": { 17 | "@vuepress/plugin-google-analytics": "^1.9.7", 18 | "@vuepress/plugin-last-updated": "^1.9.7", 19 | "@vuepress/plugin-medium-zoom": "^1.9.7", 20 | "vuepress": "^1.9.2", 21 | "vuepress-plugin-code-copy": "^1.0.6", 22 | "vuepress-plugin-sitemap": "^2.3.1" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /docs/src/.vuepress/components/Foo/Bar.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 16 | -------------------------------------------------------------------------------- /docs/src/.vuepress/components/OtherComponent.vue: -------------------------------------------------------------------------------- 1 | 4 | -------------------------------------------------------------------------------- /docs/src/.vuepress/components/demo-component.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 16 | -------------------------------------------------------------------------------- /docs/src/.vuepress/enhanceApp.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Client app enhancement file. 3 | * 4 | * https://v1.vuepress.vuejs.org/guide/basic-config.html#app-level-enhancements 5 | */ 6 | 7 | export default ({ 8 | Vue, // the version of Vue being used in the VuePress app 9 | options, // the options for the root Vue instance 10 | router, // the router instance for the app 11 | siteData // site metadata 12 | }) => { 13 | // ...apply enhancements for the site. 14 | } 15 | -------------------------------------------------------------------------------- /docs/src/.vuepress/public/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Allow: / 3 | Sitemap: https://heumsi.github.io/apache-airflow-tutorials-for-beginner/sitemap.xml 4 | -------------------------------------------------------------------------------- /docs/src/.vuepress/styles/index.styl: -------------------------------------------------------------------------------- 1 | /** 2 | * Custom Styles here. 3 | * 4 | * ref：https://v1.vuepress.vuejs.org/config/#index-styl 5 | */ 6 | 7 | .home .hero img 8 | max-width 450px!important 9 | 10 | .page 11 | p, ul, ol 12 | line-height: 1.8 13 | 14 | img 15 | display: block 16 | margin: 40px auto 17 | 18 | img + em 19 | margin-top -20px 20 | margin-bottom 40px 21 | display block 22 | text-align center 23 | color gray 24 | font-size 0.85em 25 | -------------------------------------------------------------------------------- /docs/src/.vuepress/styles/palette.styl: -------------------------------------------------------------------------------- 1 | /** 2 | * Custom palette here. 3 | * 4 | * ref：https://v1.vuepress.vuejs.org/zh/config/#palette-styl 5 | */ 6 | 7 | $accentColor = #3eaf7c 8 | $textColor = #2c3e50 9 | $borderColor = #eaecef 10 | $codeBgColor = #282c34 11 | -------------------------------------------------------------------------------- /docs/src/architecture/01-overview/README.md: -------------------------------------------------------------------------------- 1 | # 개요 2 | 3 | ![img.png](./img.png) 4 | 5 | Airflow는 크게 다음과 같은 컴포넌트들로 구성되어 있습니다. 6 | 7 | - **DAG Directory** 8 | - 파이썬으로 작성된 DAG 파일을 저정하는 공간입니다. 9 | - **Scheduler** 10 | - Airflow의 가장 중요한 부분으로, 다양한 DAG Run과 Task들을 스케쥴링 및 오케스트레이션합니다. 11 | - 또한 하나의 DAG Run이 전체 시스템을 압도하지 않도록 각 DAG Run의 실행 횟수를 제한하기도 합니다. 12 | - Meta Database에 DAG 정보 및 DAG Run에 대해 저장합니다. 13 | - **Executor** 14 | - Scheduler 내부의 구성 요소입니다. 15 | - Scheduler가 작업을 조정하는 동안 Executor는 실제로 작업을 실행합니다. 16 | - Sequential, Local, Celery, Kubernetes 등 Executor에는 여러 종류가 있습니다. (기본 값은 Sequential Executor 입니다.) 17 | - **Worker** 18 | - Executor에 의해 만들어지며 Task를 실제로 실행하는 프로세스입니다. 19 | - Executor의 종류에 따라 Worker는 쓰레드, 프로세스, 파드가 될 수 있습니다. 20 | - **Meta Database** 21 | - DAG, 해당 실행 및 사용자, 역할 및 연결과 같은 기타 Airflow 구성에 대한 메타데이터를 저장합니다. 22 | - **Webserver** 23 | - Airflow의 Web UI 입니다. 24 | - Meta Database로 부터 DAG 정보를 읽어와 DAG 정보 및 DAG Run의 상태를 확인하고 실행할 수 있습니다. 25 | -------------------------------------------------------------------------------- /docs/src/architecture/01-overview/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/architecture/01-overview/img.png -------------------------------------------------------------------------------- /docs/src/architecture/02-dag-directory/README.md: -------------------------------------------------------------------------------- 1 | # DAG Directory 2 | 3 | ## 개념 4 | 5 | DAG 파일을 저장하는 공간입니다. `dag_folder` 혹은 `dags_folder` 로도 불립니다. 6 | 기본적으로 `$AIRFLOW_HOME/dags/` 가 DAG Directory로 설정되어 있습니다. 7 | 8 | DAG을 작성한 후 DAG Directory에 저장하면, Airflow Scheduler가 주기적으로 DAG Directory를 스캔한후 DAG을 파싱해갑니다. 9 | 이 때 DAG 인스턴스만 파싱해가므로, 사실 DAG 그 자체가 아니라, DAG을 구성에 사용할 다른 코드들(Custom Operator나 기타 Python Code들)을 이 안에 넣어도 무방합니다. 10 | 11 | 또한 다음처럼 DAG Directory 내부가 여러 디렉터리로 구성되어 있어도 Airflow Scheduler는 모든 디렉토리 내부를 탐색하며 파일을 스캔합니다. 12 | 13 | ``` 14 | # $AIRFLOW_HOME/dags/ 15 | 16 | project_a/ 17 | a_1.py 18 | a_2.py 19 | project_b/ 20 | b_1.py 21 | b_2.py 22 | ``` 23 | 24 | DAG의 개수가 많아지면 DAG Directory 내부를 어떻게 구조화 할지에 대해서 잘 생각해봐야 합니다. 25 | 26 | ## 설정 27 | 28 | `$AIRFLOW_HOME/airflow.cfg` 파일에 다음과 같은 설정들을 지정해볼 수 있습니다. 29 | 30 | ### `dags_folder` 31 | 32 | ``` 33 | [core] 34 | dags_folder = ./dags 35 | ``` 36 | 37 | `$AIRFLOW_HOME` 기준 `./dags` 경로를 DAG Directory로 설정합니다. 38 | 39 | ## `.airflowignore` 40 | 41 | `.airflowignore` 는 DAG Directory 내부에 Airflow Scheduler가 스캔하지 않고 무시해야할 파일 목록을 담는 파일입니다. 42 | 43 | 예를 들어 DAG Directory 내부에 다음과 같이 `.airflowignore` 를 정의했다고 해봅시다. 44 | 45 | ``` 46 | # $AIRFLOW_HOME/dags/.airflowignore 47 | 48 | a.py 49 | b/ 50 | ``` 51 | 52 | 이 경우 Airflow Scheduler는 DAG Directory 내 `a.py`와 `b` 디렉토리 내 모든 파일을 스캔하지 않고 넘어갑니다. 53 | 이처럼 `.airflowignore` 는 스캔하지 않아야할 파일이나 디렉토리가 있을 경우 유용하게 사용할 수 있습니다. 54 | -------------------------------------------------------------------------------- /docs/src/architecture/04-executor/01-sequential-executor/README.md: -------------------------------------------------------------------------------- 1 | # Sequential Executor 2 | 3 | Sequential Executor는 Airflow를 설치하고 별다른 설정을 하지 않으면 기본으로 설정되는 Executor입니다. 4 | 한편 Airflow의 Meta Database가 기본 값인 SQLite로 설정되어있을 때 사용할 수 있는 유일한 Executor이기도 합니다. 5 | 6 | 이 Executor는 한 번에 하나의 Task Instance만 실행합니다. 이 때문에 운영 환경에서는 사용하지 않는 편입니다. 7 | 8 | ![](https://miro.medium.com/max/1400/1*QAioF8YX5xEFb92hHHoFEQ.png) 9 | *출처: https://insaid.medium.com/executors-in-airflow-2357675b8284* 10 | -------------------------------------------------------------------------------- /docs/src/architecture/04-executor/02-local-executor/README.md: -------------------------------------------------------------------------------- 1 | # Local Executor 2 | 3 | Local Executor는 Task Instance를 병렬적으로(Parallel) 여러 개 실행할 수 있습니다. 4 | 각 Task Instance는 별도의 프로세스에서 실행되는데 이렇게 Task Instance를 실행하는 프로세스를 워커라고 부릅니다. 5 | 한편, `$AIRFLOW_HOME/airflow.cfg` 설정 파일에서 `parallelism` 설정 값에 따라 Local Executor의 동작 과정이 조금 달라집니다. 6 | 7 | `parallelism` 의 값이 0인 경우, Unlimited Parallelism 이라고 부르며, 한 번에 실행할 수 있는 Task Instance의 개수에 제한이 없습니다. 8 | 즉 `QUEUED` 상태인 모든 Task Instance가 각각의 프로세스로 생성되어 실행되며, 실행이 완료돠면 프로세스도 종료됩니다. 9 | 10 | `parallelism` 의 값이 0이 아닌 경우, Limited Parallelism 이라고 부르며, Local Executor 실행 초기에 `parallelism` 개수 만큼의 워커 프로세스를 미리 만들어둡니다. 11 | 이후 `QUEUED` 상태인 Task Instance들은 이 워커 프로세스들 내에서 실행됩니다. 12 | 13 | `parallelism` 의 기본 설정 값은 32로 되어있습니다. (즉, Airflow에서 별도의 설정을 하지 않고 Local Executor를 사용하면 Local Executor는 Limited Parallelism의 형태로 동작합니다.) 14 | 15 | ![](https://miro.medium.com/max/1400/1*juMYCvszj1lcycoEOvGVAw.png) 16 | *출처: https://insaid.medium.com/executors-in-airflow-2357675b8284* 17 | -------------------------------------------------------------------------------- /docs/src/architecture/04-executor/03-celery-executor/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/architecture/04-executor/03-celery-executor/img.png -------------------------------------------------------------------------------- /docs/src/architecture/04-executor/04-kubernetes-executor/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Executor 2 | 3 | Kubernetes Executor는 Kuberentes 환경에서 사용할 수 있는 Executor로, 파드를 통해 Task Instance를 실행합니다. 4 | 5 | Kuberentes Executor는 `QUEUED` 상태인 Task Instance를 찾아 Kubernetes API Server에 해당 Task Instance를 실행할 파드 생성을 요청합니다. 6 | 즉 Task Instance 별로 파드(Worker 파드라 부릅니다)가 생기고 실행되며, 실행이 완료된 파드는 삭제됩니다. 7 | 8 | ![](https://miro.medium.com/max/1400/1*GuXasZX0WVhj8cHmMshIBg.png) 9 | *출처: https://insaid.medium.com/executors-in-airflow-2357675b8284* 10 | 11 | Kubernetes Executor를 사용하면 Task Instance를 독립적인 컨테이너 환경에서 실행할 수 있고, 리소스(CPU, Memory) 역시 다르게 줄 수 있어, 12 | Airflow를 매우 유연하게 사용할 수 있습니다. 또한 필요한 Worker 파드를 동적으로 생성하고 삭제하기 떄문에 전체 리소스 사용량도 매우 효율적이게 됩니다. 13 | 14 | 일반적으로 Kubernetes는 어느 정도 규모가 있는 Airflow 운영과 궁합이 잘 맞다고 알려져 있으며, 15 | Kubernetes 환경에서 Airflow 사용 시 Kubernetes Executor가 일반적으로 권장되고 있습니다. 16 | (다만 Kubernetes 환경은 분산 노드 환경이므로, DAG Direcotory를 별도로 Sync 해야하고, 17 | 로그 설정 역시 Remote Logging으로 해야하는 등 추가 설정해야할 것들이 조금 더 있습니다.) 18 | 19 | ![img.png](./img.png) 20 | *출처: https://airflow.apache.org/docs/apache-airflow/stable/executor/kubernetes.html* 21 | -------------------------------------------------------------------------------- /docs/src/architecture/04-executor/04-kubernetes-executor/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/architecture/04-executor/04-kubernetes-executor/img.png -------------------------------------------------------------------------------- /docs/src/architecture/04-executor/README.md: -------------------------------------------------------------------------------- 1 | # Executor 2 | 3 | Executor는 Scheduler에서 생성하는 서브 프로세스로 Queue에 들어온 Task Instance를 실제로 실행하는 역할을 합니다. 4 | 5 | Executor에는 여러 타입이 있는데, 크게 2가지로 나눠볼 수 있습니다. 6 | 7 | - Local Executors 8 | - Task Instance를 Scheduler 프로세스 내부에서 실행합니다. 9 | - Remote Executors 10 | - Task Instance를 Scheduler 프로세스 외부에서 실행합니다. 11 | 12 | 각 Executor는 좀 더 세부적으로 다음처럼 나뉩니다. 13 | 14 | - Local Executors 15 | - Sequential Executor 16 | - Local Executor 17 | - Remote Executors 18 | - Celery Executor 19 | - Kubernetes Executor 20 | 21 | 이러한 Executor `$AIRFLOW_HOME/airflow.cfg` 에서 `executor` 설정 값으로 지정해줄 수 있습니다. (기본 값은 Sequential Executor 입니다.) 22 | 23 | ``` 24 | # $AIRFLOW_HOME/airflow.cfg 25 | 26 | executor=SequentialExecutor 27 | ``` 28 | 29 | 이제 각 Executor에 대해 좀 더 자세히 알아봅시다. 30 | 31 | :::tip 32 | 위 Executor 외에도 Debug Executor 등 소개하지 않은 Executor가 있습니다. 33 | 이 문서에서는 실제로 주로 쓰이는 Executor들에 대해서만 소개합니다. 여기서 소개하지 않은 Executor에 대한 내용은 [공식 문서](https://airflow.apache.org/docs/apache-airflow/2.2.3/executor/index.html)를 확인하세요. 34 | ::: 35 | -------------------------------------------------------------------------------- /docs/src/architecture/05-meta-database/airflow_meta_db_erd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/architecture/05-meta-database/airflow_meta_db_erd.png -------------------------------------------------------------------------------- /docs/src/architecture/06-webserver/README.md: -------------------------------------------------------------------------------- 1 | # Webserver 2 | 3 | ## 개념 4 | 5 | Webserver는 Meta Database와 통신하며 DAG, DAG Runs, Task Instance, Variables, Connections 등의 데이터를 가져와 6 | 웹에서 보여주고 유저와 상호작용 할 수 있게 합니다. 7 | 8 | ![img.png](./img.png) 9 | 10 | 사실 Webserver가 없어도 Airflow의 DAG이 실행 또는 스케줄링은 여전히 되기 때문에, 다른 컴포넌트들 보다는 장애에 있어서 덜 민감한 컴포넌트이기는 합니다. 11 | 12 | ## 설정 13 | 14 | Meta Database와 관련하여 다음과 같은 설정 값들이 있습니다. 15 | 16 | ### `default_dag_run_display_number` 17 | 18 | UI(DAG 상세 화면)에서 보여줄 DAG Runs의 수입니다. 기본 값은 25개 입니다. 19 | 20 | ### `default_ui_timezone` 21 | 22 | UI에서 보여줄 기본 타임존입니다. `"Asia/Seoul"` 과 같은 IANA 타임존 형태로 값을 지정하면 됩니다. 기본 값은 UTC 입니다. 23 | 24 | 이 값을 빈 문자열(`""`)로 두면, `default_timezone` 설정 값을 따라갑니다. 25 | 26 | ### `page_size` 27 | 28 | UI(대시보드)에서 보여줄 DAG 리스트의 페이지 사이즈입니다. 기본 값은 100개 입니다. 29 | 30 | ### `worker_class` 31 | 32 | Webserver Worker(gunicorn)이 사용할 Worker 클래스입니다. `sync`, `eventlet`, `gevent` 중 하나를 지정하면 됩니다. 33 | 기본 값은 `sync` 입니다. 34 | 35 | 이 값은 Webserver Worker 클래스를 바꿔 UI 속도를 올리고 싶을 때 유용할 수 있습니다. 36 | 37 | :::tip 38 | 여기서 소개된 것들보다 훨씬 더 많은 설정 값들이 있습니다. 이에 대한 내용은 [공식 문서](https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#webserver)에서 확인하세요. 39 | ::: 40 | -------------------------------------------------------------------------------- /docs/src/architecture/06-webserver/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/architecture/06-webserver/img.png -------------------------------------------------------------------------------- /docs/src/architecture/README.md: -------------------------------------------------------------------------------- 1 | # 아키텍처 2 | 3 | Airflow는 어떤 컴포넌트들로 이루어져있고, 각 역할이 무엇인지에 대해 알아봅시다. 4 | -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/01-simple-task-dependency/README.md: -------------------------------------------------------------------------------- 1 | # 간단한 Task 의존성 작성하기 2 | 3 | ## Graph View 4 | 5 | 다음과 같은 간단한 Task 의존성을 가지는 DAG을 작성해봅시다. 6 | 7 | ![image-20220122121951658](./image-20220122121951658.png) 8 | 9 | ## Code 10 | 11 | <<< @/../my-airflow-project/dags/01_writing_various_task_flows/01_simple_tasks.py{28} 12 | 13 | - `>>` 연결로 순차적인 Task 의존성을 만듭니다. 아주 간단합니다. 14 | 15 | ## Web UI 16 | 17 | ![image-20220122122006564](./image-20220122122006564.png) 18 | -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/01-simple-task-dependency/image-20220122121951658.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/01-simple-task-dependency/image-20220122121951658.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/01-simple-task-dependency/image-20220122122006564.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/01-simple-task-dependency/image-20220122122006564.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/02-parallel-tasks-flow/image-20220122115223321.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/02-parallel-tasks-flow/image-20220122115223321.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/02-parallel-tasks-flow/image-20220122122532097.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/02-parallel-tasks-flow/image-20220122122532097.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/03-parallel-task-list-flow/README.md: -------------------------------------------------------------------------------- 1 | # 병렬 Task 리스트 의존성 작성하기 2 | 3 | ## Graph View 4 | 5 | 다음과 같은 Task 의존성을 가지는 DAG을 작성해봅시다. 6 | 7 | ![image-20220122150241425](./image-20220122150241425.png) 8 | 9 | ## Code 10 | 11 | <<< @/../my-airflow-project/dags/01_writing_various_task_flows/03_parallel_task_list.py{26-29,33} 12 | 13 | - `List`에 Task Instance를 담아 Task 의존성 정의하는 부분에 사용할 수 있습니다. 14 | - 의존성 앞 뒤에 있는 Task Instance에 `List` 내 모든 Task Instance가 연결됩니다. 15 | 16 | ## Web UI 17 | 18 | ![image-20220122150332463](./image-20220122150332463.png) 19 | -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/03-parallel-task-list-flow/image-20220122150241425.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/03-parallel-task-list-flow/image-20220122150241425.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/03-parallel-task-list-flow/image-20220122150332463.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/03-parallel-task-list-flow/image-20220122150332463.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/04-conditional-task-flow-with-branch/README.md: -------------------------------------------------------------------------------- 1 | # 조건에 따라 다른 Task 의존성 작성하기 2 | 3 | ## Graph View 4 | 5 | 다음처럼 특정 조건(Branch)에 따라 분기 처리되어 실행되는 Task Instance가 달라지는 의존성을 가지는 DAG을 작성해봅시다. 6 | 7 | ![image-20220123003919600](./image-20220123003919600.png) 8 | 9 | ## Code 10 | 11 | <<< @/../my-airflow-project/dags/01_writing_various_task_flows/04_conditional_tasks_with_branch.py{7,26-27,32-38,40} 12 | 13 | - `32-38` 라인에 `BranchPythonOperator` 로 Task Instance를 정의합니다. 14 | - `select_random` 함수를 `task_ids` 의 파라미터 값인 `[t2.task_id, t3.task_id]` 를 함께 넘기며 호출합니다. 15 | - `26-27` 라인의 `select_random` 함수는 `task_ids` 리스트 값중 하나를 랜덤으로 선택해 반환합니다. 16 | - 즉 `t2.task_id`, `t3.task_id` 중 하나가 선택됩니다. 17 | - 선택된 `task`가 실행됩니다. 18 | - `40` 라인에 `BranchPythonOperator` 로 정의한 Task Instance 뒤에 분기에 따라 실행할 Task Instance를 담은 리스트가 있습니다. 19 | 20 | 조건에 따라 다른 Task 의존성을 갖는 방법을 정리해보면 다음과 같습니다. 21 | 22 | - **조건에 따라 분기 처리해야 할 Task Instance들을 `List`에 담아 `BranchPythonOperator` Task Instance에 의존하도록 설정합니다.,** 23 | - **`BranchPythonOperator` Task Instance는 이 `List` 에 담긴 Task Instance 중 하나의 Task Instance의 `task_id`를 반환합니다.** 24 | 25 | ## Web UI 26 | 27 | ![image-20220122171121697](./image-20220122171121697.png) 28 | -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/04-conditional-task-flow-with-branch/image-20220122171121697.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/04-conditional-task-flow-with-branch/image-20220122171121697.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/04-conditional-task-flow-with-branch/image-20220123003919600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/04-conditional-task-flow-with-branch/image-20220123003919600.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/05-conditional-task-flow-with-depends-on-past/README.md: -------------------------------------------------------------------------------- 1 | # 직전 DAG Run의 Task 상태에 따라 Task 실행 여부를 결정하기 2 | 3 | ## Graph View 4 | 5 | 다음처럼 간단한 Task 의존성을 가지는 DAG을 작성해볼 것입니다. 6 | 7 | ![image-20220123003635284](./image-20220123003635284.png) 8 | 9 | 간단해 보이지만 하나 다른 점은 **이전 DAG Run의 Task Instance의 상태가 성공인 경우에만 현재 DAG Run의 Task Instance을 실행**한다는 것입니다. 10 | Grahp View에서는 이러한 내용이 잘 보이지 않으니 Tree View를 살펴보겠습니다. 11 | 12 | ## Tree View 13 | 14 | 작성할 DAG을 실행하면 다음과 같은 Tree View를 얻게 됩니다. 15 | 16 | ![image-20220122220345864](./image-20220122220345864.png) 17 | 18 | 자세히 살펴보면, 첫 번째 DAG Run의 두 번째 Task Instnace가 실패한 상태이기 때문에 두 번째 DAG Run의 두 번째 Task Instance가 실행되지 않은 상태임을 알 수 있습니다. 19 | (세 번째 Task Instance 역시 두 번째 Task Instance에 의존이 있기 때문에 실행되지 않았습니다.) 20 | 21 | ## Code 22 | 23 | <<< @/../my-airflow-project/dags/01_writing_various_task_flows/05_conditional_tasks_with_depends_on_past.py{28-30} 24 | 25 | 26 | - `PythonOperator`의 `depends_on_past` 파라미터 값을 `True`로 넘겨줍니다. 27 | - `depends_on_past` 의 기본 값은 `False` 입니다. 28 | - `PythonOperator` 뿐 아니라 제공되는 모든 `Operator`에 `depends_on_past`가 존재합니다. 29 | 30 | :::tip 31 | `depends_on_past=True` 를 다음처럼 `default_args` 에 넣어주면, 모든 Task에 대해 직전 DAG Run의 Task 각각의 상태에 따라 실행할 수 있게 됩니다. 32 | 33 | ```python 34 | with DAG( 35 | ... 36 | default_args={ 37 | ... 38 | "depends_on_past": True 39 | }, 40 | ... 41 | ) as dag: 42 | ... 43 | ``` 44 | ::: 45 | 46 | ## Web UI 47 | 48 | ![image-20220122225639740](./image-20220122225639740.png) 49 | -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/05-conditional-task-flow-with-depends-on-past/image-20220122220345864.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/05-conditional-task-flow-with-depends-on-past/image-20220122220345864.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/05-conditional-task-flow-with-depends-on-past/image-20220122225639740.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/05-conditional-task-flow-with-depends-on-past/image-20220122225639740.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/05-conditional-task-flow-with-depends-on-past/image-20220123003635284.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/05-conditional-task-flow-with-depends-on-past/image-20220123003635284.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/06-conditional-task-flow-with-wait-for-downstream/README.md: -------------------------------------------------------------------------------- 1 | # 직전 DAG Run의 상태에 따라 Task 실행 여부를 결정하기 2 | 3 | ## Graph View 4 | 5 | 다음처럼 간단한 Task 의존성을 가지는 DAG을 작성해볼 것입니다. 6 | 7 | ![image-20220123003728946](./image-20220123003728946.png) 8 | 9 | 간단해 보이지만 하나 다른 점은 **이전 DAG Run의 상태가 성공인 경우에만 현재 DAG Run을 실행**한다는 것입니다. 10 | Grahp View에서는 이러한 내용이 잘 보이지 않으니 Tree View를 살펴보겠습니다. 11 | 12 | :::tip 13 | 직전 내용과 다른 점은, 직전 내용은 이전 DAG Run의 Task Instance 상태가 주요 요인이었다면, 이번 내용은 **이전 DAG Run 그 자체의 상태**가 주요 요인이라는 것입니다. 14 | 즉 실행 단위가 Task Instance가 아닌 DAG Run 입니다. 15 | ::: 16 | 17 | ## Tree View 18 | 19 | 작성할 DAG을 실행하면 다음과 같은 Tree View를 얻게 됩니다. 20 | 21 | ![image-20220122225546239](./image-20220122225546239.png) 22 | 23 | 자세히 살펴보면, 첫 번째 DAG Run이 실패한 상태고, 두 번째 DAG Run의 실행 중 상태이지만, 어떠한 Task Instance도 실행되지 않은 것을 볼 수 있습니다. 24 | 두 번째 DAG Run의 모든 Task Instance가 실행되지 않은 이유는 직전 DAG Run이 실패한 상태이기 때문입니다. 25 | 26 | ## Code 27 | 28 | <<< @/../my-airflow-project/dags/01_writing_various_task_flows/06_conditional_tasks_with_wait_for_downstream.py{27-29} 29 | 30 | 31 | - `PythonOperator`의 `wait_for_downstream` 파라미터 값을 `True`로 넘겨줍니다. 32 | - `wait_for_downstream` 의 기본 값은 `False` 입니다. 33 | - `PythonOperator` 뿐 아니라 제공되는 모든 `Operator`에 `depends_on_past`가 존재합니다. 34 | 35 | :::tip 36 | `wait_for_downstream=True` 를 다음처럼 `default_args` 에 넣어주면, 모든 Task에 대해 직전 DAG Run의 Task 각각의 상태에 따라 실행할 수 있게 됩니다. 37 | 38 | ```python 39 | with DAG( 40 | ... 41 | default_args={ 42 | ... 43 | "wait_for_downstream": True 44 | }, 45 | ... 46 | ) as dag: 47 | ... 48 | ``` 49 | ::: 50 | 51 | ## Web UI 52 | 53 | ![image-20220122225611854](./image-20220122225611854.png) 54 | -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/06-conditional-task-flow-with-wait-for-downstream/image-20220122225546239.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/06-conditional-task-flow-with-wait-for-downstream/image-20220122225546239.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/06-conditional-task-flow-with-wait-for-downstream/image-20220122225611854.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/06-conditional-task-flow-with-wait-for-downstream/image-20220122225611854.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/06-conditional-task-flow-with-wait-for-downstream/image-20220123003728946.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/06-conditional-task-flow-with-wait-for-downstream/image-20220123003728946.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/07-conditional-task-flow-with-trigger-rule/README.md: -------------------------------------------------------------------------------- 1 | # Trigger Rule에 따라 Task 실행 여부를 결정하기 2 | 3 | ## Graph View 4 | 5 | 다음처럼 간단한 Task 의존성을 가지는 DAG을 작성해볼 것입니다. 6 | 7 | ![image-20220123003032553](./image-20220123003032553.png) 8 | 9 | 간단해 보이지만 조금 특이한 점은 `task_3`가 실패한 상태임에도 `task_4`가 실행되어 성공한 상태로 남아있다는 것입니다. 10 | Task Instance의 실행 조건을 **Trigger Rule**이라고 부르는데, 기본적으로 의존하는 모든 Task Instance가 성공 상태여야 해당 Task Instance가 실행됩니다. 11 | 12 | 여기서는 Trigger Rule의 기본 값이 아닌 다른 옵션으로 수정해보겠습니다. 13 | 14 | ## Code 15 | 16 | <<< @/../my-airflow-project/dags/01_writing_various_task_flows/07_conditional_tasks_with_trigger_rule.py{30-32} 17 | 18 | - `PythonOperator`의 `trigger_rule` 파라미터 값으로 `"one_success"`을 넘겨주었습니다. 19 | - 이는 의존하는 Task Instance의 상태가 단 하나라도 성공 상태이면 해당 Task Instance를 실행한다는 의미입니다. 20 | - `trigger_rule`의 기본 값은 `"all_success"` 입니다. 21 | 22 | :::tip 23 | 더 다양한 Trigger Rule 옵션을 보고 싶으면 [공식 문서](https://airflow.apache.org/docs/apache-airflow/stable/concepts/dags.html?highlight=trigger%20rule#trigger-rules)를 확인해보세요. 24 | ::: 25 | 26 | ## Web UI 27 | 28 | ![image-20220123003131411](./image-20220123003131411.png) 29 | -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/07-conditional-task-flow-with-trigger-rule/image-20220123003032553.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/07-conditional-task-flow-with-trigger-rule/image-20220123003032553.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/07-conditional-task-flow-with-trigger-rule/image-20220123003131411.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/07-conditional-task-flow-with-trigger-rule/image-20220123003131411.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/README.md: -------------------------------------------------------------------------------- 1 | # SubDAG 만들기 2 | 3 | :::warning 4 | SubDAG은 현재(Airflow 2.2.3 기준) 일반적으로 사용이 권장되지는 않습니다. (현재는 Task Group 사용을 권장합니다.) 5 | 다만 기존에 작성된 DAG 파일을 보게되는 경우 간혹 등장하기 때문에, 한번 둘러본다는 차원으로 보시면 좋습니다. 6 | ::: 7 | 8 | ## Graph View 9 | 10 | 다음과 같은 Task 의존성을 가지는 DAG을 작성해볼 것입니다. 11 | 12 | ![image-20220123154556951](./image-20220123154556951.png) 13 | 14 | 조금 특이한 점은 가운데에 `subdag` Task의 색이 다른 Task들과 다르다는 것입니다. 15 | 이 Task는 SubDAG 이라 부르는 Task로, 이름 그대로 DAG 안에 존재하는 또 다른 DAG 입니다. 16 | 17 | 이 `subdag` Task는 다음과 같이 또 다른 DAG으로 구성되어 있습니다. 18 | 19 | ![image-20220123154746623](./image-20220123154746623.png) 20 | 21 | 이처럼 SubDAG은 DAG 내 존재하는 하위 DAG입니다. 22 | 이제 이를 어떻게 작성하는지 살펴봅시다. 23 | 24 | ## Code 25 | 26 | <<< @/../my-airflow-project/dags/01_writing_various_task_flows/08_subdag.py{6,14-37,56-58} 27 | 28 | - `56-58` 라인에서 `SubDagOperator` 를 통해 SubDAG을 정의합니다. 29 | - 이 때 `subdag` 파라미터를 통해 `get_subdag` 함수를 호출합니다. 30 | - `subdag` 파라미터에 넘길 함수는 `DAG` 객체를 반환하는 함수여야 합니다. 31 | - `14-37` 라인에 `get_subdag` 함수가 정의되어 있습니다. 32 | - 일반적인 `DAG` 작성과 동일합니다. 이 DAG은 SubDAG이 됩니다. 33 | - 하나 주의해야할 점은 SubDAG의 `dag_id`는 `{parent_dag_id}.` 를 prefix로 가져야 합니다. 만약 prefix 갖지 않으면 Scheduler에서 DAG 파일을 파싱할 때 에러를 뱉습니다. 34 | - 이 함수는 `DAG` 인스턴스를 반환합니다. 35 | 36 | :::warning 37 | SubDAG을 사용할 때는 좀 더 주의해야할 사항들이 있습니다. 자세한 내용은 [공식 문서](https://airflow.apache.org/docs/apache-airflow/stable/concepts/dags.html?highlight=subdag#subdags)에서 확인해주세요. 38 | ::: 39 | 40 | ## Web UI 41 | 42 | ![image-20220123154845833](./image-20220123154845833.png) 43 | 44 | ![image-20220123154859276](./image-20220123154859276.png) 45 | 46 | ![image-20220123154910866](./image-20220123154910866.png) 47 | -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/image-20220123154556951.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/image-20220123154556951.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/image-20220123154746623.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/image-20220123154746623.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/image-20220123154845833.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/image-20220123154845833.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/image-20220123154859276.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/image-20220123154859276.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/image-20220123154910866.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/01-writing-various-tasks-dependencies/08-subdag/image-20220123154910866.png -------------------------------------------------------------------------------- /docs/src/dags/01-writing-various-tasks-dependencies/README.md: -------------------------------------------------------------------------------- 1 | # 다양한 Task 의존성 작성하기 2 | 3 | 다양한 Task 간 의존성 작성에 대해 알아봅시다. 4 | -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123162512663.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123162512663.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123162839172.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123162839172.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123162936827.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123162936827.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123163031694.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123163031694.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123163048155.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123163048155.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123163102483.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123163102483.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123163941903.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/01-trigger-dagrun/image-20220123163941903.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img_5.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/02-dependencies-between-dags/02-waiting-tasks-in-another-dagrun/img_6.png -------------------------------------------------------------------------------- /docs/src/dags/02-dependencies-between-dags/README.md: -------------------------------------------------------------------------------- 1 | # DAG 간 의존성 작성하기 2 | 3 | DAG 간 의존성을 작성하는 방법을 알아봅시다. 4 | -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/01-task-groups/README.md: -------------------------------------------------------------------------------- 1 | # UI에 Task들을 Group으로 표현하기 2 | 3 | ## Web UI 4 | 5 | Task들을 Group화 하여 보여줄 수 있는 **Task Group** 이라는 개념이 있습니다. 6 | Task Group을 사용하면 그룹화된 Task들이 Graph View에서 다음과 같이 `group_1` Task 처럼 보이게 됩니다. 7 | 8 | ![img.png](./img.png) 9 | 10 | `group_1` Task를 클릭하면 다음처럼 그룹화된 Task들을 볼 수 있습니다. 11 | 12 | ![img_1.png](./img_1.png) 13 | 14 | ## Code 15 | 16 | <<< @/../my-airflow-project/dags/03_visualization_and_documentation_dags/01_task_groups.py{6,27-33} 17 | 18 | - `with` 구문과 같이 `TaskGroup` 인스턴스로 Task Group을 정의합니다. 19 | - `with` 구문 내에 정의된 Task Instance들과 의존성은 모두 하나의 Task Group으로 그룹화 됩니다. 20 | 21 | :::tip 22 | Task Group은 단순히 Web UI에서 그룹화되어서 보이는 것일 뿐, 동작에는 영향을 주거나 별다른 의미는 없습니다. 23 | ::: 24 | -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/01-task-groups/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/03-visualization-and-documentation-dags/01-task-groups/img.png -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/01-task-groups/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/03-visualization-and-documentation-dags/01-task-groups/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/02-labels/README.md: -------------------------------------------------------------------------------- 1 | # UI에 Task 간 Edge에 Label 추가하기 2 | 3 | ## Web UI 4 | 5 | Graph View에서 다음처럼 Task 간 Edge에 Label을 부여할 수 있습니다. 6 | 7 | ![img.png](./img.png) 8 | 9 | ## Code 10 | 11 | <<< @/../my-airflow-project/dags/03_visualization_and_documentation_dags/02_labels.py{6,26,28,31} 12 | -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/02-labels/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/03-visualization-and-documentation-dags/02-labels/img.png -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/03-documentation/README.md: -------------------------------------------------------------------------------- 1 | # UI에 DAG 문서화 추가하기 2 | 3 | ## Web UI 4 | 5 | 다음과 같이 DAG에 대한 문서화를 작성할 수 있습니다. 6 | 7 | ![img.png](./img.png) 8 | 9 | 또한 다음처럼 Task에 대한 문서화도 작성할 수 있습니다. 10 | 11 | ![img_1.png](./img_1.png) 12 | 13 | ![img_2.png](./img_2.png) 14 | 15 | ## Code 16 | 17 | <<< @/../my-airflow-project/dags/03_visualization_and_documentation_dags/03_documentation.py{19-24,31} 18 | 19 | :::tip 20 | 마크다운(md) 뿐 아니라 json이나 rst형식도 가능합니다. 자세한 내용은 [공식 문서](https://airflow.apache.org/docs/apache-airflow/stable/concepts/dags.html#dag-task-documentation)를 확인해보세요. 21 | ::: 22 | -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/03-documentation/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/03-visualization-and-documentation-dags/03-documentation/img.png -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/03-documentation/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/03-visualization-and-documentation-dags/03-documentation/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/03-documentation/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/03-visualization-and-documentation-dags/03-documentation/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/04-save-dag-image/README.md: -------------------------------------------------------------------------------- 1 | # DAG을 이미지로 저장하기 2 | 3 | ## CLI 4 | 5 | 다음과 같은 명령어로 DAG을 이미지로 저장할 수 있습니다. 6 | 7 | ```bash 8 | $ airflow dags show {dag_id} --save {image_name.png}` 9 | ``` 10 | 11 | 예를 들면 `dag_id`가 `hello_world` 인 DAG을 `output.png` 이미지로 저장하는 명령어는 다음과 같습니다. 12 | 13 | ```bash 14 | $ airflow dags show hello_world --save output.png 15 | ``` 16 | 17 | 생성된 `output.png` 는 다음과 같습니다. 18 | 19 | ![](./output.png) 20 | -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/04-save-dag-image/output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/03-visualization-and-documentation-dags/04-save-dag-image/output.png -------------------------------------------------------------------------------- /docs/src/dags/03-visualization-and-documentation-dags/README.md: -------------------------------------------------------------------------------- 1 | # DAG 시각화와 문서화 하기 2 | 3 | DAG을 시각화하고 문서화하는 방법에 대해 알아봅시다. 4 | -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/01-bash-operator/README.md: -------------------------------------------------------------------------------- 1 | # BashOperator 2 | 3 | `BashOperator`는 `bash` 커맨드를 실행하는 Operator입니다. 4 | 5 | ## Graph View 6 | 7 | 다음처럼 간단한 Task 의존성을 가지는 DAG을 작성해봅시다. 8 | 9 | ![img.png](./img.png) 10 | 11 | ## Code 12 | 13 | <<< @/../my-airflow-project/dags/04_using_various_operators/01_bash_operator.py{4,20-23} 14 | 15 | - `BashOperator`로 `bash` 커맨드를 실행하는 Task Instance를 생성합니다. 16 | - `bash_command` 파라미터에 실행할 셸 커맨드를 넘기면 됩니다. 17 | - `env` 파라미터에 `Dict` 형태로 환경변수를 주입할 수 있습니다. 18 | 19 | ## Web UI 20 | 21 | 실행 결과를 확인하면 다음과 같습니다. 22 | 23 | ![img_2.png](./img_2.png) 24 | 25 | Task Instance 로그를 통해 의도한대로 bash 커맨드가 잘 동작한 것을 확인할 수 있습니다. 26 | 27 | ![img_1.png](./img_1.png) 28 | 29 | ![img_3.png](./img_3.png) 30 | -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/01-bash-operator/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/01-bash-operator/img.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/01-bash-operator/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/01-bash-operator/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/01-bash-operator/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/01-bash-operator/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/01-bash-operator/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/01-bash-operator/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/02-python-operator/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/02-python-operator/img.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/02-python-operator/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/02-python-operator/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/02-python-operator/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/02-python-operator/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/02-python-operator/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/02-python-operator/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/02-python-operator/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/02-python-operator/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/02-python-operator/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/02-python-operator/img_5.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/03-python-virtualenv-operator/README.md: -------------------------------------------------------------------------------- 1 | # PythonVirtualenvOperator 2 | 3 | `PythonVirtualenvOperator` 는 `virtualenv` 로 가상환경 진입 및 패키지 설치 후 파이썬 Callable 객체(여기엔 함수도 포함됩니다)를 실행하는 Operator입니다. 4 | `virtualenv` 로 격리된 공간에서 필요한 패키지를 설치할 수 있다는 점 외에는 `PythonOperator`와 사용법과 동작은 동일합니다. 5 | 6 | ## Graph View 7 | 8 | 다음처럼 간단한 Task 의존성을 가지는 DAG을 작성해봅시다. 9 | 10 | ![img.png](./img.png) 11 | 12 | ## Code 13 | 14 | <<< @/../my-airflow-project/dags/04_using_various_operators/03_python_virtualenv_operator.py{4,32-38,40-47} 15 | 16 | - `PythonVirtualenvOperator`로 가상환경을 만든 뒤 패키지를 설치하고 파이썬 Callable 객체를 실행합니다. 17 | - `python_callable` 파라미터에 실행할 파이썬 Callable 객체를 넘깁니다. 18 | - `requirements` 파라미터에 가상환경에 설치할 파이썬 패키지 목록을 `List[str]` 타입으로 넘깁니다. 19 | - `python_version` 파라미터로 가상환경에 사용할 파이썬 버전을 지정할 수도 있습니다. 20 | 21 | ## Web UI 22 | 23 | 다음처럼 잘 실행된 것을 로그를 통해 확인할 수 있습니다. 24 | 25 | ![img_1.png](./img_1.png) 26 | 27 | ![img_2.png](./img_2.png) 28 | 29 | ![img_3.png](./img_3.png) 30 | -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/03-python-virtualenv-operator/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/03-python-virtualenv-operator/img.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/03-python-virtualenv-operator/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/03-python-virtualenv-operator/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/03-python-virtualenv-operator/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/03-python-virtualenv-operator/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/03-python-virtualenv-operator/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/03-python-virtualenv-operator/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/04-branch-python-operator/README.md: -------------------------------------------------------------------------------- 1 | # BranchPythonOperator 2 | 3 | `BranchPythonOperator` 는 특정 조건에 따라 의존성 흐름에 분기를 줄 수 있는 Operator입니다. 4 | 이전 글 [조건에 따라 다른 Task 의존성 작성하기](/dags/01-writing-various-tasks-dependencies/04-conditional-task-flow-with-branch/)에서 이미 한 번 살펴봤으므로 5 | 여기에서 자세한 내용은 생략하겠습니다. 6 | 7 | ## Graph View 8 | 9 | ![image-20220123003919600](./image-20220123003919600.png) 10 | 11 | ## Code 12 | 13 | <<< @/../my-airflow-project/dags/01_writing_various_task_flows/04_conditional_tasks_with_branch.py{7,26-27,32-38,40} 14 | 15 | ## Web UI 16 | 17 | ![image-20220122171121697](./image-20220122171121697.png) 18 | -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/04-branch-python-operator/image-20220122171121697.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/04-branch-python-operator/image-20220122171121697.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/04-branch-python-operator/image-20220123003919600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/04-branch-python-operator/image-20220123003919600.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/05-dummy-operator/README.md: -------------------------------------------------------------------------------- 1 | # DummyOperator 2 | 3 | `DummyOperator` 는 아무 실행도 하지 않는 Operator입니다. 간혹 Task 간 의존성 흐름 내 필요한 경우에 사용됩니다. 4 | 5 | ## Graph View 6 | 7 | 다음과 같은 Task 의존성을 가지는 DAG을 작성해봅시다. 8 | 9 | ![img.png](./img.png) 10 | 11 | - `first_task_0` ~ `first_task_4` Task Instance는 병렬로 실행됩니다. 12 | - 이 중 하나가 먼저 성공했다고 하더라도 가운데 `dummy` Task Instance가 이전의 모든 Task Instance가 성공되기를 기다리기 때문에, 13 | `first_task_0` ~ `first_task_4` 가 모두 성공해야 그 다음 `second_task_0` ~ `second_task_4` Task Instance를 한 번에 병렬로 실행하게 됩니다. 14 | 15 | ## Code 16 | 17 | <<< @/../my-airflow-project/dags/04_using_various_operators/04_dummy_operator.py{5,29} 18 | 19 | ## Web UI 20 | 21 | 다음처럼 잘 실행된 것을 로그를 통해 확인할 수 있습니다. 22 | 23 | ![img_1.png](./img_1.png) 24 | 25 | `DummyOperator` Task Instance는 어떤 로그도 출력하지 않았습니다. 26 | 27 | ![img_2.png](./img_2.png) 28 | -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/05-dummy-operator/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/05-dummy-operator/img.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/05-dummy-operator/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/05-dummy-operator/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/05-dummy-operator/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/05-dummy-operator/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/06-email-operator/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/06-email-operator/img.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/06-email-operator/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/06-email-operator/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/06-email-operator/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/06-email-operator/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/06-email-operator/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/06-email-operator/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/06-email-operator/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/06-email-operator/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/06-email-operator/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/06-email-operator/img_5.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/07-custom-operator/README.md: -------------------------------------------------------------------------------- 1 | # Custom Operator 2 | 3 | Airflow Operator는 직접 Custom 하게 작성할 수 있습니다. 4 | 작성자가 직접 Custom Operator를 만드는 방법에 대해 알아봅시다. 5 | 6 | ## Graph View 7 | 8 | 다음과 같은 간단한 Task 의존성을 가지는 DAG을 작성할 것입니다. 9 | 10 | ![img.png](./img.png) 11 | 12 | 이 떄 `print_sql` Task는 제가 직접 작성한 Custom Operator로 만들어진 Task입니다. 13 | 14 | ## Code 15 | 16 | <<< @/../my-airflow-project/dags/04_using_various_operators/06_custom_operator.py{5,9-22,37-41} 17 | 18 | - `9-22` 라인에 `PrintSQLOperator` 라는 이름의 Custom Operator를 정의합니다. 19 | - Custom Operator는 `BaseOperator` 를 상속받아야 합니다. 20 | - 또한 `execute` 메서드를 정의해야 합니다. 21 | - Custom Operator로 정의된 Task Instance 실행 시 `execute` 함수 내용이 실행됩니다. 22 | - 따라서 Custom Operator의 메인 로직이 `execute` 함수 내에 담겨야 합니다. 23 | - `ui_color` 파라미터로 Web UI에서의 색을 지정할 수 있습니다. (위 Graph View에서 검은색) 24 | - `ui_fgcolor` 파라미터로 Web UI에서의 색을 지정할 수 있습니다. (위 Graph View에서 흰색) 25 | - `template_fields` 파라미터로 템플릿 문법을 지원할 필드를 지정할 수 있습니다. 26 | - `template_ext` 파라미터로 템플리 문법을 지원할 파일 확장자를 지정할 수 있습니다. 27 | - `37-41` 라인에 `PrintSQLOperator` 로 Task Instance를 정의합니다. 28 | - `name` 파라미터는 `PrintSQLOperator.template_fields` 에 포함되기 때문에 템플릿 변수를 받을 수 있습니다. 29 | - `sql` 파라미터는 `06_custom_operator.sql` 라는 파일을 받고 있습니다. 30 | - `06_custom_operator.sql` 은 다음과 같은 외부 파일입니다. 31 | - <<< @/../my-airflow-project/dags/04_using_various_operators/06_custom_operator.sql 32 | - `.sql`이 `PrintSQLOperator.template_ext` 에 포함되기 때문에 `.sql` 파일을 템플릿 문법으로 읽어올 수 있습니다. 33 | 34 | 35 | ## Web UI 36 | 37 | 실행 결과는 다음과 같습니다. 38 | 39 | ![img_1.png](./img_1.png) 40 | 41 | ![img_2.png](./img_2.png) 42 | -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/07-custom-operator/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/07-custom-operator/img.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/07-custom-operator/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/07-custom-operator/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/07-custom-operator/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/04-using-various-operators/07-custom-operator/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/04-using-various-operators/README.md: -------------------------------------------------------------------------------- 1 | # 다양한 Operator 사용해보기 2 | 3 | Airflow가 제공해주는 기본 Operator들을 사용해봅시다. 4 | -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/01-xcoms/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/01-xcoms/img.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/01-xcoms/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/01-xcoms/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/01-xcoms/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/01-xcoms/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/01-xcoms/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/01-xcoms/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/01-xcoms/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/01-xcoms/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/02-variables/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/02-variables/img.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/02-variables/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/02-variables/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/02-variables/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/02-variables/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/02-variables/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/02-variables/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/02-variables/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/02-variables/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/02-variables/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/02-variables/img_5.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/02-variables/img_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/02-variables/img_7.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/02-variables/img_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/02-variables/img_8.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/02-variables/img_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/02-variables/img_9.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/03-connections-and-hooks/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/03-connections-and-hooks/img.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/03-connections-and-hooks/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/03-connections-and-hooks/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/03-connections-and-hooks/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/03-connections-and-hooks/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/03-connections-and-hooks/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/03-connections-and-hooks/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/03-connections-and-hooks/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/03-connections-and-hooks/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/03-connections-and-hooks/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/03-connections-and-hooks/img_5.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/03-connections-and-hooks/img_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/03-connections-and-hooks/img_6.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/04-templates/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/04-templates/img.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/04-templates/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/04-templates/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/04-templates/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/04-templates/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/04-templates/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/04-templates/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/04-templates/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/04-templates/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/04-templates/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/05-etc-features/04-templates/img_5.png -------------------------------------------------------------------------------- /docs/src/dags/05-etc-features/README.md: -------------------------------------------------------------------------------- 1 | # 그 외 다양한 기능 사용해보기 2 | 3 | 지금까지 소개하지 않은 에어플로우에서 제공해주는 다양한 기능을 살펴봅시다. 4 | -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/01-concept/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/01-concept/img.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/01-concept/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/01-concept/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/01-concept/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/01-concept/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/01-concept/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/01-concept/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/01-concept/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/01-concept/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/01-concept/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/01-concept/img_5.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/02-custom-dependencies/README.md: -------------------------------------------------------------------------------- 1 | # 커스텀하게 Task간 의존성 작성하기 2 | 3 | Taskflow API를 사용하면 Task 간 의존 관계를 명시적으로 정의해주지 않아도 자동으로 파악하여 설정되었습니다. 4 | 이는 직접 정의해줄 필요가 없어서 편리할 수 있는 반면, 의존 관계가 명시적이지 않아 불편할 수도 있습니다. 5 | 6 | 이번에는 Taskflow API에서도 명시적으로 Task간 의존 관계를 정의하는 방법에 대해 알아봅시다. 7 | 8 | ## Graph View 9 | 10 | 다음과 같은 Task 의존성을 갖는 DAG을 작성할 것입니다. 11 | 12 | ![img.png](./img.png) 13 | 14 | ## Code 15 | 16 | <<< @/../my-airflow-project/dags/06_taskflow_api/02_custom_dependencies.py{40-42,44} 17 | 18 | - `40-42` 라인을 보면 `@task` 를 붙인 함수를 호출한 뒤 그 반환 값을 저장합니다. 19 | - 이 값은 Task Instance가 됩니다. 20 | - `44` 라인에 이 Task Instnace간 의존 관계를 정의합니다. 21 | 22 | ## Web UI & Logs 23 | 24 | DAG을 실행하면 다음과 같은 화면을 얻습니다. 25 | 26 | ![img_1.png](./img_1.png) 27 | 28 | 각 Task Instance의 로그를 살펴보면 다음과 같습니다. 29 | 30 | ### `print_hello` 31 | 32 | ![img_2.png](./img_2.png) 33 | 34 | ``` 35 | [2022-01-31, 15:43:04 UTC] {logging_mixin.py:109} INFO - hello 36 | ``` 37 | 38 | ### `get_first_word_from_param_and_return` 39 | 40 | ![img_3.png](./img_3.png) 41 | 42 | ``` 43 | [2022-01-31, 15:43:06 UTC] {logging_mixin.py:109} INFO - welcome 44 | ``` 45 | 46 | ### `get_last_word_from_param` 47 | 48 | ![img_4.png](./img_4.png) 49 | 50 | ``` 51 | [2022-01-31, 15:43:51 UTC] {logging_mixin.py:109} INFO - tutorials 52 | ``` 53 | -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/02-custom-dependencies/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/02-custom-dependencies/img.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/02-custom-dependencies/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/02-custom-dependencies/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/02-custom-dependencies/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/02-custom-dependencies/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/02-custom-dependencies/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/02-custom-dependencies/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/02-custom-dependencies/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/02-custom-dependencies/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/03-with-operators/README.md: -------------------------------------------------------------------------------- 1 | # Operator와 함께 사용하기 2 | 3 | Taskflow API와 기존의 방식인 Operator를 같이 사용하는 방법을 알아봅시다. 4 | 5 | ## Graph View 6 | 7 | 다음과 같은 Task 의존성을 갖는 DAG을 작성할 것입니다. 8 | 9 | ![img.png](./img.png) 10 | 11 | ## Code 12 | 13 | <<< @/../my-airflow-project/dags/06_taskflow_api/03_with_operators.py{3-4,20-37,39-49,51} 14 | 15 | - 전체적으로 Operator를 사용하는 방식과 큰 차이가 없습니다. 16 | - `27-33` 번 라인에 `@task` 데코레이터로 Task를 정의한 부분이 있습니다. 17 | - `46` 번 라인에 이 Task를 Task Instance로 정의합니다. 18 | - 이 때 `task_1.output` 으로 `task_1` 의 `return` 값을 가져올 수 있습니다. 19 | - `51` 번 라인에는 Task 간 의존 관계를 정의합니다. 크게 다른 부분은 없습니다. 20 | 21 | ## Web UI & Logs 22 | 23 | DAG을 실행하면 다음과 같은 화면을 얻습니다. 24 | 25 | ![img_1.png](./img_1.png) 26 | 27 | 각 Task Instance의 로그를 살펴보면 다음과 같습니다. 28 | 29 | ### `get_first_word_from_param_and_return` 30 | 31 | ![img_2.png](./img_2.png) 32 | 33 | ``` 34 | [2022-01-31, 15:52:14 UTC] {logging_mixin.py:109} INFO - welcome 35 | ``` 36 | 37 | ### `get_last_word_from_param_and_return` 38 | 39 | ![img_3.png](./img_3.png) 40 | 41 | ``` 42 | [2022-01-31, 15:57:04 UTC] {logging_mixin.py:109} INFO - tutorials 43 | ``` 44 | 45 | ### `print_param` 46 | 47 | ![img_4.png](./img_4.png) 48 | 49 | ``` 50 | [2022-01-31, 15:57:07 UTC] {logging_mixin.py:109} INFO - welcome to airflow tutorials 51 | ``` 52 | -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/03-with-operators/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/03-with-operators/img.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/03-with-operators/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/03-with-operators/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/03-with-operators/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/03-with-operators/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/03-with-operators/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/03-with-operators/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/03-with-operators/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/03-with-operators/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/04-virtualenv/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/04-virtualenv/img.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/04-virtualenv/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/04-virtualenv/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/04-virtualenv/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/04-virtualenv/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/05-docker/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/05-docker/img.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/05-docker/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/05-docker/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/05-docker/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/06-taskflow-api/05-docker/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/06-taskflow-api/README.md: -------------------------------------------------------------------------------- 1 | # Taskflow API 사용해보기 2 | 3 | Taskflow API를 사용하여 DAG을 작성하는 방법에 대해 알아봅시다. 4 | -------------------------------------------------------------------------------- /docs/src/dags/07-providers/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/07-providers/img.png -------------------------------------------------------------------------------- /docs/src/dags/07-providers/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/07-providers/img_1.png -------------------------------------------------------------------------------- /docs/src/dags/07-providers/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/07-providers/img_2.png -------------------------------------------------------------------------------- /docs/src/dags/07-providers/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/07-providers/img_3.png -------------------------------------------------------------------------------- /docs/src/dags/07-providers/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/07-providers/img_4.png -------------------------------------------------------------------------------- /docs/src/dags/07-providers/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/07-providers/img_5.png -------------------------------------------------------------------------------- /docs/src/dags/07-providers/img_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/07-providers/img_6.png -------------------------------------------------------------------------------- /docs/src/dags/07-providers/img_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/07-providers/img_7.png -------------------------------------------------------------------------------- /docs/src/dags/07-providers/img_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/dags/07-providers/img_8.png -------------------------------------------------------------------------------- /docs/src/dags/README.md: -------------------------------------------------------------------------------- 1 | # DAG 2 | 3 | DAG 작성에 대해 좀 더 알아봅시다. 4 | -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_1.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_10.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_11.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_12.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_13.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_14.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_15.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_16.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_17.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_18.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_19.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_2.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_20.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_3.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_4.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_5.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_6.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_7.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_8.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/01-gcp/img_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/01-cloud-vendor/01-gcp/img_9.png -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/02-aws/README.md: -------------------------------------------------------------------------------- 1 | # AWS - MWAA 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/src/deployment/01-cloud-vendor/README.md: -------------------------------------------------------------------------------- 1 | # Cloud Managed 서비스 이용하기 2 | 3 | Airflow를 가장 쉽게 이용할 수 있는 방법 중 하나는 Cloud Managed 서비스를 이용하는 것입니다. 4 | 5 | GCP(Google Cloud Platform)에서는 Cloud Composer라는 이름으로, AWS에서는 MWAA(Amazon Managed Workflows for Apache Airflow)라는 이름으로 Cloud 기반의 Managed Airflow를 제공합니다. 6 | -------------------------------------------------------------------------------- /docs/src/deployment/02-native/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/02-native/img.png -------------------------------------------------------------------------------- /docs/src/deployment/02-native/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/02-native/img_1.png -------------------------------------------------------------------------------- /docs/src/deployment/02-native/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/02-native/img_2.png -------------------------------------------------------------------------------- /docs/src/deployment/03-docker/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/03-docker/img.png -------------------------------------------------------------------------------- /docs/src/deployment/03-docker/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/03-docker/img_1.png -------------------------------------------------------------------------------- /docs/src/deployment/03-docker/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/03-docker/img_2.png -------------------------------------------------------------------------------- /docs/src/deployment/04-docker-compose/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/04-docker-compose/img.png -------------------------------------------------------------------------------- /docs/src/deployment/04-docker-compose/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/04-docker-compose/img_1.png -------------------------------------------------------------------------------- /docs/src/deployment/04-docker-compose/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/04-docker-compose/img_2.png -------------------------------------------------------------------------------- /docs/src/deployment/05-kubernetes/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/05-kubernetes/img.png -------------------------------------------------------------------------------- /docs/src/deployment/05-kubernetes/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/deployment/05-kubernetes/img_1.png -------------------------------------------------------------------------------- /docs/src/deployment/README.md: -------------------------------------------------------------------------------- 1 | # 배포 2 | 3 | Airflow를 배포하는 다양한 방법에 대해 알아봅시다. 4 | -------------------------------------------------------------------------------- /docs/src/faq/README.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/src/getting-started/01-installation/README.md: -------------------------------------------------------------------------------- 1 | # 설치하기 2 | 3 | ## 실행 환경 소개 4 | 5 | 다음 환경에서 설치 및 실습을 진행합니다. 6 | 7 | - 파이썬 3.8.x 8 | - macOS Big Sur 11.2.3 9 | 10 | :::tip 11 | 파이썬은 [pyenv](https://github.com/pyenv/pyenv)로 쉽게 설치할 수 있습니다. 12 | 파이썬 버전이 다르면 이 문서와 조금 다른 결과가 나올 수 있습니다. 13 | ::: 14 | 15 | ## 가상 환경 만들기 16 | 17 | Airflow 프로젝트를 위한 디렉토리를 하나 만들고 진입합니다. 18 | 19 | ```bash 20 | # Airflow 프로젝트를 위한 디렉토리를 생성합니다. 21 | $ mkdir my-airflow-project 22 | 23 | # 생성한 디렉토리에 진입합니다. 24 | $ cd my-airflow-project 25 | ``` 26 | 27 | 간단하게 파이썬 기본 가상 환경 모듈 `venv` 로 가상 환경을 만들고 진입합니다. 28 | 29 | ```bash 30 | # 가상 환경을 생성합니다. 31 | $ python -m venv .venv 32 | 33 | # 생성한 가상 환경에 진입합니다. 34 | $ source .venv/bin/activate 35 | $ (.venv) # 가상 환경에 진입하면 셸에 (.venv) 와 같은 표시가 생깁니다. (사용하는 셀 마다 다를 수 있습니다.) 36 | ``` 37 | 38 | 이후 모든 셸 명령어 실행은 가상 환경에서 진행됩니다. 다만 편의상 `(.venv)` 를 표시하진 않겠습니다. 39 | 40 | ## 패키지 설치 41 | 42 | 먼저 패키지 매니저 도구인 `pip` 를 다음처럼 최신 버전으로 업데이트 해줍니다. 43 | 44 | ```bash 45 | $ pip install pip --upgrade 46 | ``` 47 | 48 | 이제 다음처럼 `pip` 를 통해 `apache-airflow` 패키지를 설치합니다. 49 | 50 | ```bash 51 | $ pip install apache-airflow==2.2.3 52 | ``` 53 | 54 | ::: tip 55 | 저는 현 시점 가장 최신 버전인 `2.2.3` 버전을 받았습니다. 56 | Airflow의 최신 버전은 공식 [Github Repo](https://github.com/apache/airflow)에서 확인하실 수 있습니다. 57 | ::: 58 | 59 | 설치가 완료되었습니다! 이제 본격적으로 Airflow를 사용해봅시다. 60 | -------------------------------------------------------------------------------- /docs/src/getting-started/02-deployment/image-20220109204105305.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/02-deployment/image-20220109204105305.png -------------------------------------------------------------------------------- /docs/src/getting-started/02-deployment/image-20220109204147240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/02-deployment/image-20220109204147240.png -------------------------------------------------------------------------------- /docs/src/getting-started/02-deployment/image-20220109204211383.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/02-deployment/image-20220109204211383.png -------------------------------------------------------------------------------- /docs/src/getting-started/02-deployment/image-20220109204654304.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/02-deployment/image-20220109204654304.png -------------------------------------------------------------------------------- /docs/src/getting-started/02-deployment/image-20220109204736758.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/02-deployment/image-20220109204736758.png -------------------------------------------------------------------------------- /docs/src/getting-started/03-writing-a-dag/image-20220110232821353.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/03-writing-a-dag/image-20220110232821353.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220111233617011.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220111233617011.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220111234022226.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220111234022226.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112000007675.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112000007675.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112001053445.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112001053445.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112001907336.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112001907336.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112002034847.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112002034847.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112002145920.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112002145920.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112002706381.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112002706381.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112002811494.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112002811494.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112003422433.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112003422433.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112003515590.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112003515590.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112003918763.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112003918763.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112004005070.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112004005070.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112004132225.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112004132225.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112005139504.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112005139504.png -------------------------------------------------------------------------------- /docs/src/getting-started/04-running-a-dag/image-20220112005238029.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heumsi/apache-airflow-tutorials-for-beginner/23f8ee0490e4d7f5c559c13e49bfb55049a19675/docs/src/getting-started/04-running-a-dag/image-20220112005238029.png -------------------------------------------------------------------------------- /docs/src/getting-started/05-outro/README.md: -------------------------------------------------------------------------------- 1 | # 마치며 2 | 3 | Airflow의 설치와 실행, DAG 작성 그리고 실행까지 Airflow를 사용하는 전반적인 흐름과 방법에 대해 배웠습니다. 4 | 지금까지 진행한 내용을 정리해보면 다음과 같습니다. 5 | 6 | 1. 가상환경을 만들고 Airflow 패키지를 설치한다. 7 | 2. Airflow를 배포한다. 8 | - Airflow 프로젝트 루트를 환경변수 `AIRFLOW_HOME`로 지정한다. 9 | - Airflow Meta Database를 초기화한다. 10 | - Airflow Admin 권한의 계정을 생성한다. 11 | - Webserver를 실행한다. 12 | - Scheduler를 실행한다. 13 | 3. DAG을 작성한다. 14 | - `AIRFLOW_HOME` 경로에 `dags/` 디렉토리를 만든다. 15 | - 이 안에 DAG을 파이썬 모듈을 통해 작성한다. 16 | - DAG 작성은 다음으로 구성된다. 17 | - DAG 인스턴스 정의 18 | - Task 인스턴스 정의 19 | - Task 인스턴스 의존성(순서) 정의 20 | 4. DAG을 실행한다. 21 | - 브라우저에서 Webserver에 접속하여 버튼을 통해 DAG을 ON 상태로 만든다. 22 | - 실행 결과와 로그를 볼 수 있다. 23 | - Trigger 버튼을 통해 스케줄링 되지 않은 DAG Run을 즉시 만들어 실행할 수 있다. 24 | - Webserver에서 DAG을 삭제하면 Meta Database에서 해당 DAG의 실행 기록들만 삭제되고, DAG 파일 자체는 삭제되지 않는다. 25 | 26 | Airflow 사용에 대한 전체적인 흐름이 좀 느껴지셨나요? 물론 Airflow를 본격적으로 도입하고 사용하려면 더 많은 것들을 알아야 합니다. 27 | 이번 챕터에서는 Airflow를 사용하면 대충 이런 흐름이구나 정도만 느끼셨으면 충분합니다. 28 | 29 | 다음 챕터부터는 Airflow 사용에 대해 좀 더 자세하게 알아봅시다. 30 | -------------------------------------------------------------------------------- /docs/src/getting-started/README.md: -------------------------------------------------------------------------------- 1 | # 시작하기 2 | 3 | Airflow 설치와 사용법에 대해 빠르게 알아봅시다. 4 | -------------------------------------------------------------------------------- /docs/src/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | home: true 3 | heroImage: https://img1.daumcdn.net/thumb/R800x0/?scode=mtistory2&fname=https%3A%2F%2Fblog.kakaocdn.net%2Fdn%2F72Zk1%2FbtqPysahU2H%2Fdbpj3pZ651K59V8k561FrK%2Fimg.png 4 | tagline: Apache Airflow를 배워봅시다! 5 | actionText: 시작하기 → 6 | actionLink: /getting-started/ 7 | footer: Made by heumsi :) 8 | --- 9 | -------------------------------------------------------------------------------- /docs/src/operation/01-timezone/README.md: -------------------------------------------------------------------------------- 1 | # Timezone 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/src/operation/02-schedule-interval/README.md: -------------------------------------------------------------------------------- 1 | # Schedule Interval 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/src/operation/03-cluster-policies/README.md: -------------------------------------------------------------------------------- 1 | # Cluster Policies 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/src/operation/04-priority-weights/README.md: -------------------------------------------------------------------------------- 1 | # Priority Weights 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/src/operation/05-rbac/README.md: -------------------------------------------------------------------------------- 1 | # RBAC 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /docs/src/operation/06-interfaces/README.md: -------------------------------------------------------------------------------- 1 | # Interface 2 | 3 | ## Web UI 4 | 5 | TBD 6 | 7 | ## CLI 8 | 9 | TBD 10 | 11 | ## REST API 12 | 13 | TBD 14 | -------------------------------------------------------------------------------- /docs/src/operation/README.md: -------------------------------------------------------------------------------- 1 | # 운영 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /my-airflow-project/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v3.4.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - repo: local 8 | hooks: 9 | - id: black 10 | name: black 11 | types: [python] 12 | language: system 13 | entry: black 14 | - repo: local 15 | hooks: 16 | - id: isort 17 | name: isort 18 | types: [python] 19 | language: system 20 | entry: isort 21 | args: ["--profile", "black"] 22 | - repo: local 23 | hooks: 24 | - id: pycln 25 | name: pycln 26 | types: [python] 27 | language: system 28 | entry: pycln 29 | -------------------------------------------------------------------------------- /my-airflow-project/Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL:=help 2 | 3 | export AIRFLOW_HOME=. 4 | 5 | .PHONY: format 6 | format: ## 코드를 포매팅합니다. 7 | pycln . --all 8 | black . 9 | isort . --profile black 10 | 11 | .PHONY: scheduler 12 | scheduler: ## Airflow Scheduler를 실행합니다. 13 | airflow scheduler 14 | 15 | .PHONY: webserver 16 | webserver: ## Airflow Webserver를 실행합니다. 17 | airflow webserver 18 | 19 | .PHONY: init-db 20 | init-db: ## Airflow Database를 초기화합니다. 21 | airflow db init && \ 22 | airflow users create \ 23 | --username admin \ 24 | --password 1234 \ 25 | --firstname heumsi \ 26 | --lastname jeon \ 27 | --role Admin \ 28 | --email heumsi@naver.com 29 | 30 | .PHONY: run-db 31 | run-db: ## Airflow Database를 실행합니다. 32 | docker restart postgres || \ 33 | docker run --name postgres -e POSTGRES_USER=airflow -e POSTGRES_PASSWORD=1234 -p 5432:5432 postgres:13 34 | 35 | .PHONY: help 36 | help: ## 사용 가능한 커멘드를 확인합니다 37 | @echo 'usage: make [command]' 38 | @echo 39 | @echo 'command:' 40 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 41 | -------------------------------------------------------------------------------- /my-airflow-project/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | Apache Airflow tutorials for beginner 문서에서 사용한 예제를 담은 프로젝트입니다. 4 | 5 | ## Installation 6 | 7 | ```bash 8 | # 파이썬 버전을 확인합니다. 9 | $ python -V 10 | Python 3.8.12 11 | 12 | # 가상환경을 준비합니다. 13 | $ python -m venv .venv 14 | $ source .venv/bin/activate 15 | 16 | # Mac OS라면 psycopg2==2.9.3 패키지를 위해 다음 의존성을 설치해야 합니다. 17 | $ brew install postgresql 18 | 19 | # 의존성 패키지를 설치합니다. 20 | $ pip install -r requirements.txt 21 | ``` 22 | 23 | ## Process 24 | 25 | Docker가 실행된 환경이어야 합니다. 26 | 27 | ```bash 28 | # 다음 커맨드를 순서대로 입력합니다. 29 | 30 | # Meta Database를 실행합니다. (Docker Container) 31 | $ make run-db 32 | 33 | # Meta Database를 초기화 합니다. (초기 계정은 admin / 1234) 34 | $ make init-db 35 | 36 | # Scheduler를 실행합니다. 37 | $ make scheduler 38 | 39 | # Webserver를 실행합니다. 40 | $ make webserver 41 | ``` 42 | 43 | ## Commands 44 | 45 | ```bash 46 | $ make 47 | 48 | usage: make [command] 49 | 50 | command: 51 | format 코드를 포매팅합니다. 52 | scheduler Airflow Scheduler를 실행합니다. 53 | webserver Airflow Webserver를 실행합니다. 54 | init-db Airflow Database를 초기화합니다. 55 | run-db Airflow Database를 실행합니다. 56 | help 사용 가능한 커멘드를 확인합니다 57 | ``` 58 | 59 | ## Development 60 | 61 | ```bash 62 | # pre-commit을 설정합니다. 63 | $ pre-commit install 64 | ``` 65 | -------------------------------------------------------------------------------- /my-airflow-project/dags/01_writing_various_task_flows/01_simple_tasks.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from pendulum.tz.timezone import Timezone 7 | 8 | with DAG( 9 | dag_id="01_simple_tasks", 10 | description="가장 심플한 Task 의존성을 가지는 DAG 예제입니다.", 11 | default_args={ 12 | "owner": "heumsi", 13 | "retries": 1, 14 | "retry_delay": timedelta(minutes=1), 15 | }, 16 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 17 | schedule_interval="@once", 18 | tags=["examples", "01_writing_various_task_flows"], 19 | ) as dag: 20 | 21 | def dump() -> None: 22 | sleep(3) 23 | 24 | task_1 = PythonOperator(task_id="task_1", python_callable=dump) 25 | task_2 = PythonOperator(task_id="task_2", python_callable=dump) 26 | task_3 = PythonOperator(task_id="task_3", python_callable=dump) 27 | 28 | task_1 >> task_2 >> task_3 29 | -------------------------------------------------------------------------------- /my-airflow-project/dags/01_writing_various_task_flows/02_parallel_tasks.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from pendulum.tz.timezone import Timezone 7 | 8 | with DAG( 9 | dag_id="02_parallel_tasks", 10 | description="병렬적인 Task 의존성을 가지는 DAG 예제입니다.", 11 | default_args={ 12 | "owner": "heumsi", 13 | "retries": 1, 14 | "retry_delay": timedelta(minutes=1), 15 | }, 16 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 17 | schedule_interval="@once", 18 | tags=["examples", "01_writing_various_task_flows"], 19 | ) as dag: 20 | 21 | def dump() -> None: 22 | sleep(3) 23 | 24 | task_1 = PythonOperator(task_id="task_1", python_callable=dump) 25 | task_2 = PythonOperator(task_id="task_2", python_callable=dump) 26 | task_3 = PythonOperator(task_id="task_3", python_callable=dump) 27 | task_4 = PythonOperator(task_id="task_4", python_callable=dump) 28 | task_5 = PythonOperator(task_id="task_5", python_callable=dump) 29 | 30 | task_1 >> task_2 >> task_5 31 | task_3 >> task_4 >> task_5 32 | -------------------------------------------------------------------------------- /my-airflow-project/dags/01_writing_various_task_flows/03_parallel_task_list.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from pendulum.tz.timezone import Timezone 7 | 8 | with DAG( 9 | dag_id="03_parellel_task_list", 10 | description="병렬적인 Task를 리스트로 가지는 DAG 예제입니다.", 11 | default_args={ 12 | "owner": "heumsi", 13 | "retries": 1, 14 | "retry_delay": timedelta(minutes=1), 15 | }, 16 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 17 | schedule_interval="@once", 18 | tags=["examples", "01_writing_various_task_flows"], 19 | ) as dag: 20 | 21 | def dump() -> None: 22 | sleep(3) 23 | 24 | start_task = PythonOperator(task_id="start_task", python_callable=dump) 25 | 26 | task_list = [] 27 | for i in range(5): 28 | task = PythonOperator(task_id=f"task_{i}", python_callable=dump) 29 | task_list.append(task) 30 | 31 | end_task = PythonOperator(task_id="end_task", python_callable=dump) 32 | 33 | start_task >> task_list >> end_task 34 | -------------------------------------------------------------------------------- /my-airflow-project/dags/01_writing_various_task_flows/04_conditional_tasks_with_branch.py: -------------------------------------------------------------------------------- 1 | import random 2 | from datetime import datetime, timedelta 3 | from time import sleep 4 | from typing import List 5 | 6 | from airflow import DAG 7 | from airflow.operators.python import BranchPythonOperator, PythonOperator 8 | from pendulum.tz.timezone import Timezone 9 | 10 | with DAG( 11 | dag_id="04_conditional_tasks_with_branch", 12 | description="Task 의존성에 Branch를 통해 조건문을 가지는 DAG 예제입니다.", 13 | default_args={ 14 | "owner": "heumsi", 15 | "retries": 1, 16 | "retry_delay": timedelta(minutes=1), 17 | }, 18 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 19 | schedule_interval="@once", 20 | tags=["examples", "01_writing_various_task_flows"], 21 | ) as dag: 22 | 23 | def dump() -> None: 24 | sleep(3) 25 | 26 | def select_random(task_ids: List[str]) -> str: 27 | return random.choice(task_ids) 28 | 29 | t1 = PythonOperator(task_id="task_1", python_callable=dump) 30 | t2 = PythonOperator(task_id="task_2", python_callable=dump) 31 | t3 = PythonOperator(task_id="task_3", python_callable=dump) 32 | branch = BranchPythonOperator( 33 | task_id="branch", 34 | python_callable=select_random, 35 | op_kwargs={ 36 | "task_ids": [t2.task_id, t3.task_id], 37 | }, 38 | ) 39 | 40 | t1 >> branch >> [t2, t3] 41 | -------------------------------------------------------------------------------- /my-airflow-project/dags/01_writing_various_task_flows/05_conditional_tasks_with_depends_on_past.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from airflow.utils.dates import days_ago 7 | 8 | with DAG( 9 | dag_id="05_conditional_tasks_with_depends_on_past", 10 | description="Task 의존성에 depends_on_past를 통해 조건문을 가지는 DAG 예제입니다.", 11 | default_args={ 12 | "owner": "heumsi", 13 | "retries": 0, 14 | "retry_delay": timedelta(minutes=1), 15 | }, 16 | start_date=days_ago(2), 17 | schedule_interval=timedelta(days=1), 18 | tags=["examples", "01_writing_various_task_flows"], 19 | ) as dag: 20 | 21 | def dump() -> None: 22 | sleep(3) 23 | 24 | def must_fail() -> None: 25 | raise 26 | 27 | t1 = PythonOperator(task_id="task_1", python_callable=dump) 28 | t2 = PythonOperator( 29 | task_id="task_2", python_callable=must_fail, depends_on_past=True 30 | ) 31 | t3 = PythonOperator(task_id="task_3", python_callable=dump) 32 | 33 | t1 >> t2 >> t3 34 | -------------------------------------------------------------------------------- /my-airflow-project/dags/01_writing_various_task_flows/06_conditional_tasks_with_wait_for_downstream.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from airflow.utils.dates import days_ago 7 | 8 | with DAG( 9 | dag_id="06_conditional_tasks_with_wait_for_downstream", 10 | description="Task 의존성에 with_wait_for_downstream을 통해 조건문을 가지는 DAG 예제입니다.", 11 | default_args={ 12 | "owner": "heumsi", 13 | "retries": 0, 14 | "retry_delay": timedelta(minutes=1), 15 | }, 16 | start_date=days_ago(2), 17 | schedule_interval=timedelta(days=1), 18 | tags=["examples", "01_writing_various_task_flows"], 19 | ) as dag: 20 | 21 | def dump() -> None: 22 | sleep(3) 23 | 24 | def must_fail() -> None: 25 | raise 26 | 27 | t1 = PythonOperator( 28 | task_id="task_1", python_callable=dump, wait_for_downstream=True 29 | ) 30 | t2 = PythonOperator(task_id="task_2", python_callable=must_fail) 31 | t3 = PythonOperator(task_id="task_3", python_callable=dump) 32 | 33 | t1 >> t2 >> t3 34 | -------------------------------------------------------------------------------- /my-airflow-project/dags/01_writing_various_task_flows/07_conditional_tasks_with_trigger_rule.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from pendulum.tz.timezone import Timezone 7 | 8 | with DAG( 9 | dag_id="07_conditional_tasks_with_trigger_rule", 10 | description="Task 의존성에 trigger_rule을 통해 조건문을 가지는 DAG 예제입니다.", 11 | default_args={ 12 | "owner": "heumsi", 13 | "retries": 0, 14 | "retry_delay": timedelta(minutes=1), 15 | }, 16 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 17 | schedule_interval="@once", 18 | tags=["examples", "01_writing_various_task_flows"], 19 | ) as dag: 20 | 21 | def dump() -> None: 22 | sleep(3) 23 | 24 | def must_fail() -> None: 25 | raise 26 | 27 | t1 = PythonOperator(task_id="task_1", python_callable=dump) 28 | t2 = PythonOperator(task_id="task_2", python_callable=dump) 29 | t3 = PythonOperator(task_id="task_3", python_callable=must_fail) 30 | t4 = PythonOperator( 31 | task_id="task_4", python_callable=dump, trigger_rule="one_success" 32 | ) 33 | 34 | t1 >> [t2, t3] >> t4 35 | -------------------------------------------------------------------------------- /my-airflow-project/dags/02_dependencies_between_dags/00_to_be_triggered_by_other_dags.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from pendulum.tz.timezone import Timezone 7 | 8 | with DAG( 9 | dag_id="00_to_be_triggered_by_other_dags", 10 | description="다른 DAG에 의해 Trigger 될 DAG 예제입니다.", 11 | default_args={ 12 | "owner": "heumsi", 13 | "retries": 1, 14 | "retry_delay": timedelta(minutes=1), 15 | }, 16 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 17 | schedule_interval=None, # None으로 두었기 때문에 직접 Trigger 할 때만 DAG Run이 생성됩니다. 18 | tags=["examples", "02_dependencies_between_dags"], 19 | ) as dag: 20 | 21 | def dump() -> None: 22 | sleep(3) 23 | 24 | task_1 = PythonOperator(task_id="task_1", python_callable=dump) 25 | task_2 = PythonOperator(task_id="task_2", python_callable=dump) 26 | task_3 = PythonOperator(task_id="task_3", python_callable=dump) 27 | 28 | task_1 >> task_2 >> task_3 29 | -------------------------------------------------------------------------------- /my-airflow-project/dags/02_dependencies_between_dags/01_trigger_dagrun.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from airflow.operators.trigger_dagrun import TriggerDagRunOperator 7 | from pendulum.tz.timezone import Timezone 8 | 9 | with DAG( 10 | dag_id="01_trigger_dagrun", 11 | description="다른 DAG의 DAG Run을 Trigger하는 DAG 예제입니다", 12 | default_args={ 13 | "owner": "heumsi", 14 | "retries": 1, 15 | "retry_delay": timedelta(minutes=1), 16 | }, 17 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 18 | schedule_interval="@once", 19 | tags=["examples", "02_dependencies_between_dags"], 20 | ) as dag: 21 | 22 | def dump() -> None: 23 | sleep(3) 24 | 25 | task_1 = PythonOperator(task_id="task_1", python_callable=dump) 26 | trigger_task = TriggerDagRunOperator( 27 | task_id="trigger_task", 28 | trigger_dag_id="00_to_be_triggered_by_other_dags", 29 | conf=None, 30 | trigger_run_id=None, 31 | execution_date=None, 32 | reset_dag_run=False, 33 | wait_for_completion=False, 34 | poke_interval=60, 35 | allowed_states=["success"], 36 | failed_states=None, 37 | ) 38 | task_3 = PythonOperator(task_id="task_3", python_callable=dump) 39 | 40 | task_1 >> trigger_task >> task_3 41 | -------------------------------------------------------------------------------- /my-airflow-project/dags/02_dependencies_between_dags/02_waiting_tasks_in_another_dagrun.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from airflow.sensors.external_task import ExternalTaskSensor 7 | from pendulum.tz.timezone import Timezone 8 | 9 | with DAG( 10 | dag_id="02_waiting_tasks_in_another_dagrun", 11 | description="다른 DAG Run의 Task를 기다리는 DAG 예제입니다", 12 | default_args={ 13 | "owner": "heumsi", 14 | "retries": 1, 15 | "retry_delay": timedelta(minutes=1), 16 | }, 17 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 18 | schedule_interval="@once", 19 | tags=["examples", "02_dependencies_between_dags"], 20 | ) as dag: 21 | 22 | def dump() -> None: 23 | sleep(3) 24 | 25 | task_1 = PythonOperator(task_id="task_1", python_callable=dump) 26 | wait_for_01_trigger_dagrun_task_3 = ExternalTaskSensor( 27 | task_id="wait_for_01_trigger_dagrun.task_3", 28 | external_dag_id="01_trigger_dagrun", 29 | external_task_id="task_3", 30 | allowed_states=["success"], 31 | failed_states=None, 32 | execution_delta=None, 33 | execution_date_fn=None, 34 | check_existence=False, 35 | ) 36 | task_3 = PythonOperator(task_id="task_3", python_callable=dump) 37 | 38 | task_1 >> wait_for_01_trigger_dagrun_task_3 >> task_3 39 | -------------------------------------------------------------------------------- /my-airflow-project/dags/03_visualization_and_documentation_dags/01_task_groups.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from airflow.utils.task_group import TaskGroup 7 | from pendulum.tz.timezone import Timezone 8 | 9 | with DAG( 10 | dag_id="01_task_groups", 11 | description="TaskGroup을 가지는 DAG 예제입니다.", 12 | default_args={ 13 | "owner": "heumsi", 14 | "retries": 1, 15 | "retry_delay": timedelta(minutes=1), 16 | }, 17 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 18 | schedule_interval="@once", 19 | tags=["examples", "03_visualization_and_documentation_dags"], 20 | ) as dag: 21 | 22 | def dump() -> None: 23 | sleep(3) 24 | 25 | task_1 = PythonOperator(task_id="task_1", python_callable=dump) 26 | 27 | with TaskGroup("group_1") as task_group: 28 | task_2 = PythonOperator(task_id="task_2", python_callable=dump) 29 | task_3 = PythonOperator(task_id="task_3", python_callable=dump) 30 | task_4 = PythonOperator(task_id="task_4", python_callable=dump) 31 | 32 | task_2 >> task_4 33 | task_3 >> task_4 34 | 35 | task_5 = PythonOperator(task_id="task_5", python_callable=dump) 36 | 37 | task_1 >> task_group >> task_5 38 | -------------------------------------------------------------------------------- /my-airflow-project/dags/03_visualization_and_documentation_dags/02_labels.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from airflow.utils.edgemodifier import Label 7 | from pendulum.tz.timezone import Timezone 8 | 9 | with DAG( 10 | dag_id="02_labels", 11 | description="Label을 가지는 DAG 예제입니다.", 12 | default_args={ 13 | "owner": "heumsi", 14 | "retries": 1, 15 | "retry_delay": timedelta(minutes=1), 16 | }, 17 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 18 | schedule_interval="@once", 19 | tags=["examples", "03_visualization_and_documentation_dags"], 20 | ) as dag: 21 | 22 | def dump() -> None: 23 | sleep(3) 24 | 25 | task_1 = PythonOperator(task_id="task_1", python_callable=dump) 26 | label_1 = Label("Label 1") 27 | task_2 = PythonOperator(task_id="task_2", python_callable=dump) 28 | label_2 = Label("Label 2") 29 | task_3 = PythonOperator(task_id="task_3", python_callable=dump) 30 | 31 | task_1 >> label_1 >> task_2 >> label_2 >> task_3 32 | -------------------------------------------------------------------------------- /my-airflow-project/dags/03_visualization_and_documentation_dags/03_documentation.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.python import PythonOperator 6 | from pendulum.tz.timezone import Timezone 7 | 8 | with DAG( 9 | dag_id="03_documentation", 10 | description="문서화를 추가하는 DAG 예제입니다.", 11 | default_args={ 12 | "owner": "heumsi", 13 | "retries": 1, 14 | "retry_delay": timedelta(minutes=1), 15 | }, 16 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 17 | schedule_interval="@once", 18 | tags=["examples", "03_visualization_and_documentation_dags"], 19 | doc_md=""" 20 | ## 문서화를 추가하는 DAG 예제 21 | 22 | 코드에서 DAG 인스턴스를 생성할 때 `doc_md` 인자에 문자열을 넘겨주면 됩니다. 23 | 이처럼 마크다운으로 쓰는것이 가능합니다. 24 | """, 25 | ) as dag: 26 | 27 | def dump() -> None: 28 | sleep(3) 29 | 30 | task_1 = PythonOperator( 31 | task_id="task_1", python_callable=dump, doc_md="task에 대해서도 문서화를 추가할 수 있습니다." 32 | ) 33 | task_2 = PythonOperator(task_id="task_2", python_callable=dump) 34 | task_3 = PythonOperator(task_id="task_3", python_callable=dump) 35 | 36 | task_1 >> task_2 >> task_3 37 | -------------------------------------------------------------------------------- /my-airflow-project/dags/04_using_various_operators/01_bash_operator.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | from airflow import DAG 4 | from airflow.operators.bash import BashOperator 5 | from pendulum.tz.timezone import Timezone 6 | 7 | with DAG( 8 | dag_id="01_bash_operator", 9 | description="BashOperator를 사용하는 DAG 예제입니다.", 10 | default_args={ 11 | "owner": "heumsi", 12 | "retries": 1, 13 | "retry_delay": timedelta(minutes=1), 14 | }, 15 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 16 | schedule_interval="@once", 17 | tags=["examples", "04_using_various_operators"], 18 | ) as dag: 19 | 20 | task_1 = BashOperator(task_id="task_1", bash_command="echo Hello") 21 | task_2 = BashOperator( 22 | task_id="task_2", bash_command="echo $VAR", env={"VAR": "World!"} 23 | ) 24 | 25 | task_1 >> task_2 26 | -------------------------------------------------------------------------------- /my-airflow-project/dags/04_using_various_operators/03_python_virtualenv_operator.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | from airflow import DAG 4 | from airflow.operators.python import PythonVirtualenvOperator 5 | from pendulum.tz.timezone import Timezone 6 | 7 | with DAG( 8 | dag_id="03_python_virtualenv_operator", 9 | description="PythonVirtualenvOperator를 사용하는 DAG 예제입니다.", 10 | default_args={ 11 | "owner": "heumsi", 12 | "retries": 1, 13 | "retry_delay": timedelta(minutes=1), 14 | }, 15 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 16 | schedule_interval="@once", 17 | tags=["examples", "04_using_various_operators"], 18 | ) as dag: 19 | 20 | def print_dataframe() -> None: 21 | import pandas as pd 22 | 23 | df = pd.DataFrame({"A": [1, 2], "B": [3, 4]}) 24 | print(df.head()) 25 | 26 | def print_emoji() -> None: 27 | import emoji 28 | 29 | result = emoji.emojize("Python is :thumbs_up:") 30 | print(result) 31 | 32 | task_1 = PythonVirtualenvOperator( 33 | task_id="print_dataframe", 34 | python_callable=print_dataframe, 35 | requirements=[ 36 | "pandas==1.4.0", 37 | ], 38 | ) 39 | 40 | task_2 = PythonVirtualenvOperator( 41 | task_id="print_emoji", 42 | python_callable=print_emoji, 43 | python_version="3.9", 44 | requirements=[ 45 | "emoji==1.6.3", 46 | ], 47 | ) 48 | 49 | task_1 >> task_2 50 | -------------------------------------------------------------------------------- /my-airflow-project/dags/04_using_various_operators/04_dummy_operator.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from time import sleep 3 | 4 | from airflow import DAG 5 | from airflow.operators.dummy import DummyOperator 6 | from airflow.operators.python import PythonOperator 7 | from pendulum.tz.timezone import Timezone 8 | 9 | with DAG( 10 | dag_id="04_dummy_operator", 11 | description="DummyOperator를 사용하는 DAG 예제입니다.", 12 | default_args={ 13 | "owner": "heumsi", 14 | "retries": 1, 15 | "retry_delay": timedelta(minutes=1), 16 | }, 17 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 18 | schedule_interval="@once", 19 | tags=["examples", "04_using_various_operators"], 20 | ) as dag: 21 | 22 | def dump() -> None: 23 | sleep(3) 24 | 25 | first_tasks = [] 26 | for i in range(5): 27 | task = PythonOperator(task_id=f"first_task_{i}", python_callable=dump) 28 | first_tasks.append(task) 29 | task_2 = DummyOperator(task_id="dummy") 30 | seconds_tasks = [] 31 | for i in range(5): 32 | task = PythonOperator(task_id=f"second_task_{i}", python_callable=dump) 33 | seconds_tasks.append(task) 34 | 35 | first_tasks >> task_2 >> seconds_tasks 36 | -------------------------------------------------------------------------------- /my-airflow-project/dags/04_using_various_operators/05_email_operator.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | from airflow import DAG 4 | from airflow.operators.email import EmailOperator 5 | from pendulum.tz.timezone import Timezone 6 | 7 | with DAG( 8 | dag_id="05_email_operator", 9 | description="EmailOperator 사용하는 DAG 예제입니다.", 10 | default_args={ 11 | "owner": "heumsi", 12 | "retries": 1, 13 | "retry_delay": timedelta(minutes=1), 14 | }, 15 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 16 | schedule_interval="@once", 17 | tags=["examples", "04_using_various_operators"], 18 | ) as dag: 19 | 20 | task = EmailOperator( 21 | task_id="send_to_email", 22 | to="heumsi@naver.com", 23 | subject="Airflow에서 보낸 Email입니다.", 24 | html_content=""" 25 |

Airflow에서 보낸 Email입니다.

26 |

Airflow에서 보낸 Email 입니다!

27 | """, 28 | ) 29 | -------------------------------------------------------------------------------- /my-airflow-project/dags/04_using_various_operators/06_custom_operator.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import Any, Dict 3 | 4 | from airflow import DAG 5 | from airflow.models.baseoperator import BaseOperator 6 | from pendulum.tz.timezone import Timezone 7 | 8 | 9 | class PrintSQLOperator(BaseOperator): 10 | ui_color = "#000000" 11 | ui_fgcolor = "#ffffff" 12 | template_fields = ["name", "sql"] 13 | template_ext = [".sql"] 14 | 15 | def __init__(self, name: str, sql: str, **kwargs) -> None: 16 | super().__init__(**kwargs) 17 | self.name = name 18 | self.sql = sql 19 | 20 | def execute(self, context: Dict[str, Any]) -> None: 21 | print(f"name: {self.name}") 22 | print(f"sql: {self.sql}") 23 | 24 | 25 | with DAG( 26 | dag_id="06_custom_operator", 27 | description="CustomOperator를 사용하는 DAG 예제입니다.", 28 | default_args={ 29 | "owner": "heumsi", 30 | "retries": 0, 31 | }, 32 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 33 | schedule_interval="@once", 34 | tags=["examples", "04_using_various_operators"], 35 | ) as dag: 36 | 37 | task = PrintSQLOperator( 38 | task_id="print_sql", 39 | name="{{ task_instance.task_id }}", 40 | sql="06_custom_operator.sql", 41 | ) 42 | -------------------------------------------------------------------------------- /my-airflow-project/dags/04_using_various_operators/06_custom_operator.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM my_table WHERE created_at >= '{{ data_interval_start.strftime("%Y-%m-%d %H:%M%:%S") }}' 2 | -------------------------------------------------------------------------------- /my-airflow-project/dags/05_etc_features/03_connections_and_hooks.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from airflow import DAG 4 | from airflow.operators.python import PythonOperator 5 | from airflow.providers.postgres.hooks.postgres import PostgresHook 6 | from pendulum.tz.timezone import Timezone 7 | 8 | with DAG( 9 | dag_id="03_connections_and_hooks", 10 | description="Connection과 Hook을 활용하는 DAG 예제입니다.", 11 | default_args={ 12 | "owner": "heumsi", 13 | "retries": 0, 14 | }, 15 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 16 | schedule_interval="@once", 17 | tags=["examples", "05_etc_features"], 18 | ) as dag: 19 | 20 | postgres_hook = PostgresHook(postgres_conn_id="airflow_database") 21 | 22 | def test_connection() -> None: 23 | is_connected, message = postgres_hook.test_connection() 24 | print(is_connected) 25 | print(message) 26 | 27 | def get_variables() -> None: 28 | records = postgres_hook.get_records(sql="SELECT * FROM variable") 29 | for record in records: 30 | print(record) 31 | 32 | task_1 = PythonOperator(task_id="test_connection", python_callable=test_connection) 33 | task_2 = PythonOperator(task_id="get_variables", python_callable=get_variables) 34 | 35 | task_1 >> task_2 36 | -------------------------------------------------------------------------------- /my-airflow-project/dags/06_taskflow_api/01_concept.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from airflow.decorators import dag, task 4 | from pendulum.tz.timezone import Timezone 5 | 6 | 7 | @dag( 8 | dag_id="01_concept", 9 | description="Taskflow API를 사용하는 간단한 DAG 예제입니다.", 10 | default_args={ 11 | "owner": "heumsi", 12 | "retries": 0, 13 | }, 14 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 15 | schedule_interval="@once", 16 | tags=["examples", "06_taskflow_api"], 17 | ) 18 | def main(): 19 | """Taskflow API를 사용하는 간단한 DAG 예제입니다. 20 | 21 | 1. print_hello()와 get_first_word_from_param_and_return() 를 실행합니다. 22 | 2. get_first_word_from_param_and_return() 이 성공하면 get_last_word_from_param()를 실행합니다. 23 | """ 24 | 25 | @task 26 | def print_hello() -> None: 27 | """hello를 출력합니다.""" 28 | 29 | print("hello") 30 | 31 | @task 32 | def get_first_word_from_param_and_return(param: str) -> str: 33 | """문자열 param의 첫 번째 단어를 출력하고, param을 반환합니다.""" 34 | 35 | first_word = param.split()[0] 36 | print(first_word) # welcome 37 | return param 38 | 39 | @task 40 | def get_last_word_from_param(param: str) -> None: 41 | """문자열 param의 마지막 단어를 출력합니다.""" 42 | 43 | last_word = param.split()[-1] 44 | print(last_word) # tutorials 45 | 46 | @task 47 | def print_kwargs(**kwargs) -> None: 48 | print(kwargs) 49 | 50 | print_hello() 51 | ret = get_first_word_from_param_and_return("welcome to airflow tutorials") 52 | get_last_word_from_param(ret) 53 | print_kwargs() 54 | 55 | 56 | dag = main() 57 | -------------------------------------------------------------------------------- /my-airflow-project/dags/06_taskflow_api/02_custom_dependencies.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from airflow.decorators import dag, task 4 | from pendulum.tz.timezone import Timezone 5 | 6 | 7 | @dag( 8 | dag_id="02_custom_dependencies", 9 | description="Taskflow API의 의존성을 커스텀하게 작성하는 DAG 예제입니다.", 10 | default_args={ 11 | "owner": "heumsi", 12 | "retries": 0, 13 | }, 14 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 15 | schedule_interval="@once", 16 | tags=["examples", "06_taskflow_api"], 17 | ) 18 | def main(): 19 | @task 20 | def print_hello() -> None: 21 | """hello를 출력합니다.""" 22 | 23 | print("hello") 24 | 25 | @task 26 | def get_first_word_from_param_and_return(param: str) -> str: 27 | """문자열 param의 첫 번째 단어를 출력하고, param을 반환합니다.""" 28 | 29 | first_word = param.split()[0] 30 | print(first_word) 31 | return param 32 | 33 | @task 34 | def get_last_word_from_param(param: str) -> None: 35 | """문자열 param의 마지막 단어를 출력합니다.""" 36 | 37 | last_word = param.split()[-1] 38 | print(last_word) 39 | 40 | task_1 = print_hello() 41 | task_2 = get_first_word_from_param_and_return("welcome to airflow tutorials") 42 | task_3 = get_last_word_from_param(task_2) 43 | 44 | task_1 >> task_2 >> task_3 45 | 46 | 47 | dag = main() 48 | -------------------------------------------------------------------------------- /my-airflow-project/dags/06_taskflow_api/03_with_operators.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from airflow.decorators import dag, task 4 | from airflow.operators.python import PythonOperator 5 | from pendulum.tz.timezone import Timezone 6 | 7 | 8 | @dag( 9 | dag_id="03_with_operators", 10 | description="Operator와 Taskflow API를 같이 활용하는 DAG 예제입니다.", 11 | default_args={ 12 | "owner": "heumsi", 13 | "retries": 0, 14 | }, 15 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 16 | schedule_interval="@once", 17 | tags=["examples", "06_taskflow_api"], 18 | ) 19 | def main(): 20 | def get_first_word_from_param_and_return(param: str) -> str: 21 | """문자열 param의 첫 번째 단어를 출력하고, param을 반환합니다.""" 22 | 23 | first_word = param.split()[0] 24 | print(first_word) 25 | return param 26 | 27 | @task 28 | def get_last_word_from_param_and_return(param: str) -> str: 29 | """문자열 param의 마지막 단어를 출력하고, param을 반환합니다.""" 30 | 31 | last_word = param.split()[-1] 32 | print(last_word) 33 | return param 34 | 35 | def print_param(param: str) -> None: 36 | """문자열 param을 출력합니다""" 37 | print(param) 38 | 39 | task_1 = PythonOperator( 40 | task_id="get_first_word_from_param_and_return", 41 | python_callable=get_first_word_from_param_and_return, 42 | op_args=[ 43 | "welcome to airflow tutorials", 44 | ], 45 | ) 46 | task_2 = get_last_word_from_param_and_return(task_1.output) 47 | task_3 = PythonOperator( 48 | task_id="print_param", python_callable=print_param, op_args=[task_2] 49 | ) 50 | 51 | task_1 >> task_2 >> task_3 52 | 53 | 54 | dag = main() 55 | -------------------------------------------------------------------------------- /my-airflow-project/dags/06_taskflow_api/04_virtualenv.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from airflow.decorators import dag, task 4 | from pendulum.tz.timezone import Timezone 5 | 6 | 7 | @dag( 8 | dag_id="04_virtualenv", 9 | description="Taskflow API의 virtualenv를 활용하는 DAG 예제입니다.", 10 | default_args={ 11 | "owner": "heumsi", 12 | "retries": 0, 13 | }, 14 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 15 | schedule_interval="@once", 16 | tags=["examples", "06_taskflow_api"], 17 | ) 18 | def main(): 19 | @task.virtualenv( 20 | requirements=[ 21 | "emoji==1.6.3", 22 | ], 23 | ) 24 | def print_emoji() -> None: 25 | import emoji 26 | 27 | result = emoji.emojize("Python is :thumbs_up:") 28 | print(result) 29 | 30 | print_emoji() 31 | 32 | 33 | dag = main() 34 | -------------------------------------------------------------------------------- /my-airflow-project/dags/06_taskflow_api/05_docker.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from airflow.decorators import dag, task 4 | from pendulum.tz.timezone import Timezone 5 | 6 | 7 | @dag( 8 | dag_id="05_docker", 9 | description="Taskflow API의 docker를 활용하는 DAG 예제입니다.", 10 | default_args={ 11 | "owner": "heumsi", 12 | "retries": 0, 13 | }, 14 | start_date=datetime(2022, 1, 20, tzinfo=Timezone("Asia/Seoul")), 15 | schedule_interval="@once", 16 | tags=["examples", "06_taskflow_api"], 17 | ) 18 | def main(): 19 | @task.docker(image="python:3.9-slim-buster") 20 | def print_python_version() -> None: 21 | import sys 22 | 23 | print("Current version of Python is ", sys.version) 24 | 25 | print_python_version() 26 | 27 | 28 | dag = main() 29 | -------------------------------------------------------------------------------- /my-airflow-project/dags/hello_world.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | from airflow import DAG 4 | from airflow.operators.bash import BashOperator 5 | from airflow.operators.python import PythonOperator 6 | from airflow.utils.dates import days_ago 7 | 8 | 9 | def print_world() -> None: 10 | print("world") 11 | 12 | 13 | # with 구문으로 DAG 정의를 시작합니다. 14 | with DAG( 15 | dag_id="hello_world", # DAG의 식별자용 아이디입니다. 16 | description="My First DAG", # DAG에 대해 설명합니다. 17 | start_date=days_ago(2), # DAG 정의 기준 2일 전부터 시작합니다. 18 | schedule_interval=timedelta(days=1), # 1일을 주기로 실행합니다. 19 | tags=["examples"], 20 | ) as dag: 21 | 22 | # Task를 정의합니다. 23 | # bash 커맨드로 echo hello 를 실행합니다. 24 | t1 = BashOperator( 25 | task_id="print_hello", 26 | bash_command="echo Hello", 27 | owner="heumsi", # 이 작업의 오너입니다. 보통 작업을 담당하는 사람 이름을 넣습니다. 28 | retries=3, # 이 Task가 실패한 경우, 3번 재시도 합니다. 29 | retry_delay=timedelta(minutes=5), # 재시도하는 시간 간격은 5분입니다. 30 | ) 31 | 32 | # Task를 정의합니다. 33 | # python 함수 print_world를 실행합니다. 34 | t2 = PythonOperator( 35 | task_id="print_world", 36 | python_callable=print_world, 37 | owner="heumsi", 38 | retries=3, 39 | retry_delay=timedelta(minutes=5), 40 | ) 41 | 42 | # Task 의존성(순서)를 정합니다. 43 | # t1 실행 후 t2를 실행합니다. 44 | t1 >> t2 45 | --------------------------------------------------------------------------------