├── .gitignore ├── LICENSE ├── README.md ├── census_analysis ├── dags │ ├── census_pipeline.py │ └── snippets.py ├── data │ └── raw │ │ ├── acs_data.csv.gz │ │ └── acs_data.dta.gz └── src │ ├── analysis.py │ ├── clean_data.py │ ├── get_data.py │ ├── solutions-Analysis.ipynb │ └── solutions-Data_Prep.ipynb ├── dag_example └── simple_dag.py ├── deployments └── jupyterhub-cluster │ ├── config │ └── config.yaml │ └── image │ └── Dockerfile ├── docs ├── Makefile ├── make.bat └── source │ ├── _build │ └── html │ │ ├── _static │ │ └── uses.png │ │ ├── about.html │ │ ├── airflow-intro.html │ │ ├── first-airflow.html │ │ ├── genindex.html │ │ ├── index.html │ │ ├── objects.inv │ │ ├── pipelines.html │ │ ├── search.html │ │ ├── searchindex.js │ │ └── setup.html │ ├── _static │ ├── 12.png │ ├── 4.jpg │ ├── DAG.png │ ├── GUI.png │ ├── airflow-logo.jpeg │ ├── airflow.png │ ├── architecture.png │ ├── automate.png │ ├── automation1.jpg │ ├── azure.png │ ├── connection.png │ ├── custom.css │ ├── dag-time.png │ ├── datapyramid.png │ ├── gooddata.png │ ├── gooddata1.png │ ├── luigi.png │ ├── mssignin.png │ ├── pipeline1.png │ ├── python.png │ ├── twitter1.png │ ├── twitter2.png │ ├── twitter3.png │ └── uses.png │ ├── _templates │ └── sidebarlogo.html │ ├── about.md │ ├── airflow-intro.md │ ├── conf.py │ ├── first-airflow.md │ ├── index.rst │ ├── pipelines.md │ └── setup.rst ├── environment.yml ├── extra_tfx_example ├── dags │ ├── taxi_pipeline.py │ └── taxi_utils.py ├── data │ └── taxi_data │ │ └── data.csv └── setup │ ├── chicago_data │ ├── taxi_pipeline_simple.py │ └── taxi_utils.py │ ├── reset_env.sh │ └── setup.sh └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/README.md -------------------------------------------------------------------------------- /census_analysis/dags/census_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/census_analysis/dags/census_pipeline.py -------------------------------------------------------------------------------- /census_analysis/dags/snippets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/census_analysis/dags/snippets.py -------------------------------------------------------------------------------- /census_analysis/data/raw/acs_data.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/census_analysis/data/raw/acs_data.csv.gz -------------------------------------------------------------------------------- /census_analysis/data/raw/acs_data.dta.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/census_analysis/data/raw/acs_data.dta.gz -------------------------------------------------------------------------------- /census_analysis/src/analysis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/census_analysis/src/analysis.py -------------------------------------------------------------------------------- /census_analysis/src/clean_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/census_analysis/src/clean_data.py -------------------------------------------------------------------------------- /census_analysis/src/get_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/census_analysis/src/get_data.py -------------------------------------------------------------------------------- /census_analysis/src/solutions-Analysis.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/census_analysis/src/solutions-Analysis.ipynb -------------------------------------------------------------------------------- /census_analysis/src/solutions-Data_Prep.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/census_analysis/src/solutions-Data_Prep.ipynb -------------------------------------------------------------------------------- /dag_example/simple_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/dag_example/simple_dag.py -------------------------------------------------------------------------------- /deployments/jupyterhub-cluster/config/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/deployments/jupyterhub-cluster/config/config.yaml -------------------------------------------------------------------------------- /deployments/jupyterhub-cluster/image/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/deployments/jupyterhub-cluster/image/Dockerfile -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/source/_build/html/_static/uses.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/_static/uses.png -------------------------------------------------------------------------------- /docs/source/_build/html/about.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/about.html -------------------------------------------------------------------------------- /docs/source/_build/html/airflow-intro.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/airflow-intro.html -------------------------------------------------------------------------------- /docs/source/_build/html/first-airflow.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/first-airflow.html -------------------------------------------------------------------------------- /docs/source/_build/html/genindex.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/genindex.html -------------------------------------------------------------------------------- /docs/source/_build/html/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/index.html -------------------------------------------------------------------------------- /docs/source/_build/html/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/objects.inv -------------------------------------------------------------------------------- /docs/source/_build/html/pipelines.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/pipelines.html -------------------------------------------------------------------------------- /docs/source/_build/html/search.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/search.html -------------------------------------------------------------------------------- /docs/source/_build/html/searchindex.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/searchindex.js -------------------------------------------------------------------------------- /docs/source/_build/html/setup.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_build/html/setup.html -------------------------------------------------------------------------------- /docs/source/_static/12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/12.png -------------------------------------------------------------------------------- /docs/source/_static/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/4.jpg -------------------------------------------------------------------------------- /docs/source/_static/DAG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/DAG.png -------------------------------------------------------------------------------- /docs/source/_static/GUI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/GUI.png -------------------------------------------------------------------------------- /docs/source/_static/airflow-logo.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/airflow-logo.jpeg -------------------------------------------------------------------------------- /docs/source/_static/airflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/airflow.png -------------------------------------------------------------------------------- /docs/source/_static/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/architecture.png -------------------------------------------------------------------------------- /docs/source/_static/automate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/automate.png -------------------------------------------------------------------------------- /docs/source/_static/automation1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/automation1.jpg -------------------------------------------------------------------------------- /docs/source/_static/azure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/azure.png -------------------------------------------------------------------------------- /docs/source/_static/connection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/connection.png -------------------------------------------------------------------------------- /docs/source/_static/custom.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/custom.css -------------------------------------------------------------------------------- /docs/source/_static/dag-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/dag-time.png -------------------------------------------------------------------------------- /docs/source/_static/datapyramid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/datapyramid.png -------------------------------------------------------------------------------- /docs/source/_static/gooddata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/gooddata.png -------------------------------------------------------------------------------- /docs/source/_static/gooddata1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/gooddata1.png -------------------------------------------------------------------------------- /docs/source/_static/luigi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/luigi.png -------------------------------------------------------------------------------- /docs/source/_static/mssignin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/mssignin.png -------------------------------------------------------------------------------- /docs/source/_static/pipeline1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/pipeline1.png -------------------------------------------------------------------------------- /docs/source/_static/python.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/python.png -------------------------------------------------------------------------------- /docs/source/_static/twitter1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/twitter1.png -------------------------------------------------------------------------------- /docs/source/_static/twitter2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/twitter2.png -------------------------------------------------------------------------------- /docs/source/_static/twitter3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/twitter3.png -------------------------------------------------------------------------------- /docs/source/_static/uses.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_static/uses.png -------------------------------------------------------------------------------- /docs/source/_templates/sidebarlogo.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/_templates/sidebarlogo.html -------------------------------------------------------------------------------- /docs/source/about.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/about.md -------------------------------------------------------------------------------- /docs/source/airflow-intro.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/airflow-intro.md -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/conf.py -------------------------------------------------------------------------------- /docs/source/first-airflow.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/first-airflow.md -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/index.rst -------------------------------------------------------------------------------- /docs/source/pipelines.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/pipelines.md -------------------------------------------------------------------------------- /docs/source/setup.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/docs/source/setup.rst -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/environment.yml -------------------------------------------------------------------------------- /extra_tfx_example/dags/taxi_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/extra_tfx_example/dags/taxi_pipeline.py -------------------------------------------------------------------------------- /extra_tfx_example/dags/taxi_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/extra_tfx_example/dags/taxi_utils.py -------------------------------------------------------------------------------- /extra_tfx_example/data/taxi_data/data.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/extra_tfx_example/data/taxi_data/data.csv -------------------------------------------------------------------------------- /extra_tfx_example/setup/chicago_data/taxi_pipeline_simple.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/extra_tfx_example/setup/chicago_data/taxi_pipeline_simple.py -------------------------------------------------------------------------------- /extra_tfx_example/setup/chicago_data/taxi_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/extra_tfx_example/setup/chicago_data/taxi_utils.py -------------------------------------------------------------------------------- /extra_tfx_example/setup/reset_env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/extra_tfx_example/setup/reset_env.sh -------------------------------------------------------------------------------- /extra_tfx_example/setup/setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trallard/opendata-airflow-tutorial/HEAD/extra_tfx_example/setup/setup.sh -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | rows 3 | papermill 4 | jupyterlab 5 | apache-airflow 6 | --------------------------------------------------------------------------------