├── .env ├── .flake8 ├── .github └── workflows │ ├── airflow.yml │ └── monday-building.yml ├── .gitignore ├── Makefile ├── README.md ├── assets ├── how-to │ ├── create-dag-using-tdd.md │ └── criar-dag-usando-tdd.md └── images │ ├── airflow_first_exec_error.png │ ├── diagram.png │ ├── our_dag.png │ ├── our_initial_dag.png │ ├── output_products.png │ └── workflow_with_tests.png ├── dags └── dag_sales_pipeline.py ├── data ├── expected │ ├── agg_sales_category.csv │ ├── products_sales.csv │ ├── stg_products.csv │ └── stg_purchases_2020-01-01.csv ├── products.csv └── purchases.csv ├── docker-compose.yml ├── requirements.txt ├── sql ├── init │ ├── create_products.sql │ ├── create_products_sales.sql │ └── create_purchases.sql └── sales │ ├── agg_sales_category.sql │ ├── clean_staging_tables.sql │ ├── delete_products_sales_exec_date.sql │ ├── join_purchases_with_products.sql │ └── union_staging_to_products_sales.sql └── tests └── test_sales_pipeline.py /.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/.env -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/.flake8 -------------------------------------------------------------------------------- /.github/workflows/airflow.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/.github/workflows/airflow.yml -------------------------------------------------------------------------------- /.github/workflows/monday-building.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/.github/workflows/monday-building.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/.gitignore -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/README.md -------------------------------------------------------------------------------- /assets/how-to/create-dag-using-tdd.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/assets/how-to/create-dag-using-tdd.md -------------------------------------------------------------------------------- /assets/how-to/criar-dag-usando-tdd.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/assets/how-to/criar-dag-usando-tdd.md -------------------------------------------------------------------------------- /assets/images/airflow_first_exec_error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/assets/images/airflow_first_exec_error.png -------------------------------------------------------------------------------- /assets/images/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/assets/images/diagram.png -------------------------------------------------------------------------------- /assets/images/our_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/assets/images/our_dag.png -------------------------------------------------------------------------------- /assets/images/our_initial_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/assets/images/our_initial_dag.png -------------------------------------------------------------------------------- /assets/images/output_products.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/assets/images/output_products.png -------------------------------------------------------------------------------- /assets/images/workflow_with_tests.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/assets/images/workflow_with_tests.png -------------------------------------------------------------------------------- /dags/dag_sales_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/dags/dag_sales_pipeline.py -------------------------------------------------------------------------------- /data/expected/agg_sales_category.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/data/expected/agg_sales_category.csv -------------------------------------------------------------------------------- /data/expected/products_sales.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/data/expected/products_sales.csv -------------------------------------------------------------------------------- /data/expected/stg_products.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/data/expected/stg_products.csv -------------------------------------------------------------------------------- /data/expected/stg_purchases_2020-01-01.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/data/expected/stg_purchases_2020-01-01.csv -------------------------------------------------------------------------------- /data/products.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/data/products.csv -------------------------------------------------------------------------------- /data/purchases.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/data/purchases.csv -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | sqlAlchemy 3 | pandas 4 | psycopg2-binary -------------------------------------------------------------------------------- /sql/init/create_products.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/sql/init/create_products.sql -------------------------------------------------------------------------------- /sql/init/create_products_sales.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/sql/init/create_products_sales.sql -------------------------------------------------------------------------------- /sql/init/create_purchases.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/sql/init/create_purchases.sql -------------------------------------------------------------------------------- /sql/sales/agg_sales_category.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/sql/sales/agg_sales_category.sql -------------------------------------------------------------------------------- /sql/sales/clean_staging_tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/sql/sales/clean_staging_tables.sql -------------------------------------------------------------------------------- /sql/sales/delete_products_sales_exec_date.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/sql/sales/delete_products_sales_exec_date.sql -------------------------------------------------------------------------------- /sql/sales/join_purchases_with_products.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/sql/sales/join_purchases_with_products.sql -------------------------------------------------------------------------------- /sql/sales/union_staging_to_products_sales.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/sql/sales/union_staging_to_products_sales.sql -------------------------------------------------------------------------------- /tests/test_sales_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcosmarxm/airflow-testing-ci-workflow/HEAD/tests/test_sales_pipeline.py --------------------------------------------------------------------------------