├── airflow ├── plugins │ └── .gitkeep ├── logs │ ├── scheduler │ │ └── latest │ ├── dbt_dag │ │ ├── dbt_run │ │ │ ├── 2021-12-23T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-24T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-25T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-26T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-27T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-28T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-30T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ └── 2021-12-31T00:00:00+00:00 │ │ │ │ └── 1.log │ │ ├── test │ │ │ ├── 2021-12-31T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2022-01-01T00:00:00+00:00 │ │ │ │ ├── 1.log │ │ │ │ └── 2.log │ │ │ ├── 2022-05-14T01:31:37.396657+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2022-05-14T01:31:41.554700+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2022-05-14T01:31:44.285736+00:00 │ │ │ │ ├── 1.log │ │ │ │ └── 2.log │ │ │ ├── 2022-05-14T01:33:26.251894+00:00 │ │ │ │ ├── 1.log │ │ │ │ ├── 4.log │ │ │ │ ├── 5.log │ │ │ │ ├── 6.log │ │ │ │ ├── 7.log │ │ │ │ └── 3.log │ │ │ ├── 2021-12-24T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-23T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-25T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-26T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-27T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ ├── 2021-12-28T00:00:00+00:00 │ │ │ │ └── 1.log │ │ │ └── 2021-12-30T00:00:00+00:00 │ │ │ │ └── 1.log │ │ ├── test.jaffle_shop.int_orders │ │ │ └── 2022-05-15T03:11:14.036092+00:00 │ │ │ │ └── 1.log │ │ ├── test.jaffle_shop.fct_customer_orders │ │ │ └── 2022-05-15T03:11:14.036092+00:00 │ │ │ │ └── 1.log │ │ └── test.jaffle_shop.stg_stripe__payments │ │ │ └── 2022-05-15T03:11:14.036092+00:00 │ │ │ └── 1.log │ └── model_level_dbt_great_expectations_dag │ │ ├── dbt_test.test.jaffle_shop.assert_positive_total_for_payments │ │ └── 2022-05-17T22:12:00.138285+00:00 │ │ │ ├── 5.log │ │ │ └── 4.log │ │ ├── dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c │ │ └── 2022-05-17T22:12:00.138285+00:00 │ │ │ ├── 6.log │ │ │ ├── 2.log │ │ │ └── 5.log │ │ ├── dbt_test.test.jaffle_shop.source_unique_jaffle_shop_orders_id.8a425b2fba │ │ └── 2022-05-17T22:12:00.138285+00:00 │ │ │ └── 5.log │ │ ├── dbt_test.test.jaffle_shop.source_unique_jaffle_shop_customers_id.2777a7933e │ │ └── 2022-05-17T22:12:00.138285+00:00 │ │ │ └── 5.log │ │ ├── dbt_test.test.jaffle_shop.not_null_stg_jaffle_shop__orders_order_id.aad0410d08 │ │ └── 2022-05-17T22:12:00.138285+00:00 │ │ │ ├── 2.log │ │ │ └── 4.log │ │ ├── dbt_test.test.jaffle_shop.source_unique_stripe_payment_id.3b7989ce64 │ │ └── 2022-05-17T22:12:00.138285+00:00 │ │ │ ├── 5.log │ │ │ └── 2.log │ │ └── dbt_test.test.jaffle_shop.source_not_null_jaffle_shop_customers_id.50aa22178f │ │ └── 2022-05-17T22:12:00.138285+00:00 │ │ └── 2.log └── dags │ ├── project_level_dbt_dag.py │ └── model_level_dbt_dag.py ├── dbt └── jaffle_shop │ ├── analyses │ └── .gitkeep │ ├── macros │ ├── .gitkeep │ ├── cents_to_dollars.sql │ ├── grant_select.sql │ └── clean_stale_models.sql │ ├── seeds │ └── .gitkeep │ ├── tests │ ├── .gitkeep │ ├── assert_positive_total_for_payments.sql │ └── assert_positive_total_for_payments_daily.sql │ ├── snapshots │ └── .gitkeep │ ├── .gitignore │ ├── .user.yml │ ├── target │ ├── graph.gpickle │ ├── partial_parse.msgpack │ ├── compiled │ │ └── jaffle_shop │ │ │ ├── models │ │ │ ├── staging │ │ │ │ ├── stripe │ │ │ │ │ ├── src_stripe.yml │ │ │ │ │ │ ├── source_not_null_stripe_payment_id.sql │ │ │ │ │ │ └── source_unique_stripe_payment_id.sql │ │ │ │ │ └── stg_stripe__payments.sql │ │ │ │ └── jaffle_shop │ │ │ │ │ ├── src_jaffle_shop.yml │ │ │ │ │ ├── source_not_null_jaffle_shop_orders_id.sql │ │ │ │ │ ├── source_not_null_jaffle_shop_customers_id.sql │ │ │ │ │ ├── source_unique_jaffle_shop_orders_id.sql │ │ │ │ │ └── source_unique_jaffle_shop_customers_id.sql │ │ │ │ │ ├── stg_jaffle_shop.yml │ │ │ │ │ ├── not_null_stg_jaffle_shop__orders_order_id.sql │ │ │ │ │ ├── not_null_stg_jaffle_shop__customers_customer_id.sql │ │ │ │ │ ├── unique_stg_jaffle_shop__orders_order_id.sql │ │ │ │ │ ├── unique_stg_jaffle_shop__customers_customer_id.sql │ │ │ │ │ ├── accepted_values_stg_jaffle_sho_3ad15fa7d7f8de69cc0666c8351b51a7.sql │ │ │ │ │ ├── accepted_values_stg_jaffle_sho_016d37c576e20b0d30ec2e634d2dac96.sql │ │ │ │ │ └── relationships_stg_jaffle_shop__69c2006accf24d593d7f144684ebb92c.sql │ │ │ │ │ ├── stg_jaffle_shop__customers.sql │ │ │ │ │ └── stg_jaffle_shop__orders.sql │ │ │ ├── intermediate │ │ │ │ └── int_orders.sql │ │ │ └── marts │ │ │ │ └── core │ │ │ │ ├── fct_customer_orders.sql │ │ │ │ ├── fct_customer_orders_new.sql │ │ │ │ └── fct_customer_orders_new2.sql │ │ │ └── tests │ │ │ ├── assert_positive_total_for_payments.sql │ │ │ └── assert_positive_total_for_payments_daily.sql │ ├── run │ │ └── jaffle_shop │ │ │ ├── models │ │ │ ├── staging │ │ │ │ ├── stripe │ │ │ │ │ ├── src_stripe.yml │ │ │ │ │ │ ├── source_not_null_stripe_payment_id.sql │ │ │ │ │ │ └── source_unique_stripe_payment_id.sql │ │ │ │ │ └── stg_stripe__payments.sql │ │ │ │ └── jaffle_shop │ │ │ │ │ ├── src_jaffle_shop.yml │ │ │ │ │ ├── source_not_null_jaffle_shop_orders_id.sql │ │ │ │ │ ├── source_not_null_jaffle_shop_customers_id.sql │ │ │ │ │ ├── source_unique_jaffle_shop_orders_id.sql │ │ │ │ │ └── source_unique_jaffle_shop_customers_id.sql │ │ │ │ │ ├── stg_jaffle_shop.yml │ │ │ │ │ ├── not_null_stg_jaffle_shop__orders_order_id.sql │ │ │ │ │ ├── not_null_stg_jaffle_shop__customers_customer_id.sql │ │ │ │ │ ├── unique_stg_jaffle_shop__orders_order_id.sql │ │ │ │ │ ├── unique_stg_jaffle_shop__customers_customer_id.sql │ │ │ │ │ ├── accepted_values_stg_jaffle_sho_3ad15fa7d7f8de69cc0666c8351b51a7.sql │ │ │ │ │ ├── accepted_values_stg_jaffle_sho_016d37c576e20b0d30ec2e634d2dac96.sql │ │ │ │ │ └── relationships_stg_jaffle_shop__69c2006accf24d593d7f144684ebb92c.sql │ │ │ │ │ ├── stg_jaffle_shop__customers.sql │ │ │ │ │ └── stg_jaffle_shop__orders.sql │ │ │ ├── intermediate │ │ │ │ └── int_orders.sql │ │ │ └── marts │ │ │ │ └── core │ │ │ │ ├── fct_customer_orders.sql │ │ │ │ ├── fct_customer_orders_new.sql │ │ │ │ └── fct_customer_orders_new2.sql │ │ │ └── tests │ │ │ ├── assert_positive_total_for_payments.sql │ │ │ └── assert_positive_total_for_payments_daily.sql │ └── perf_info.json │ ├── models │ ├── staging │ │ ├── jaffle_shop │ │ │ ├── stg_jaffle_shop__customers.sql │ │ │ ├── stg_jaffle_shop__orders.sql │ │ │ ├── stg_jaffle_shop.md │ │ │ ├── src_jaffle_shop.yml │ │ │ └── stg_jaffle_shop.yml │ │ └── stripe │ │ │ ├── stg_stripe__payments.sql │ │ │ └── src_stripe.yml │ ├── intermediate │ │ └── int_orders.sql │ └── marts │ │ └── core │ │ └── fct_customer_orders.sql │ ├── README.md │ ├── profiles.yml │ └── dbt_project.yml ├── .gitignore ├── great_expectations ├── .gitignore ├── expectations │ └── .ge_store_backend_id ├── uncommitted │ ├── validations │ │ └── .ge_store_backend_id │ └── data_docs │ │ └── local_site │ │ └── static │ │ ├── images │ │ ├── favicon.ico │ │ ├── logo-long.png │ │ ├── short-logo.png │ │ ├── glossary_scroller.gif │ │ ├── iterative-dev-loop.png │ │ └── validation_failed_unexpected_values.gif │ │ ├── fonts │ │ └── HKGrotesk │ │ │ ├── HKGrotesk-Bold.otf │ │ │ ├── HKGrotesk-Italic.otf │ │ │ ├── HKGrotesk-Light.otf │ │ │ ├── HKGrotesk-Medium.otf │ │ │ ├── HKGrotesk-Regular.otf │ │ │ ├── HKGrotesk-SemiBold.otf │ │ │ ├── HKGrotesk-BoldItalic.otf │ │ │ ├── HKGrotesk-LightItalic.otf │ │ │ ├── HKGrotesk-MediumItalic.otf │ │ │ └── HKGrotesk-SemiBoldItalic.otf │ │ └── styles │ │ └── data_docs_custom_styles_template.css ├── plugins │ └── custom_data_docs │ │ └── styles │ │ └── data_docs_custom_styles.css └── checkpoints │ ├── sources │ ├── stripe__payment.yml │ ├── jaffle_shop__orders.yml │ └── jaffle_shop__customers.yml │ └── targets │ └── dev__fct_customer_orders.yml ├── assets ├── dbt.png ├── diagram.jpg ├── diagram.png ├── dbt_docs_1.png ├── dbt_docs_2.png ├── postgresql.png ├── apache_airflow.png ├── apache_superset.png ├── great-expectations.png ├── model_level_dbt_run.png ├── model_level_dbt_test.png ├── superset_daily_ltv.png ├── great_expectations_dag.png ├── model_level_dbt_dag_1.png ├── model_level_dbt_dag_2.png ├── model_level_dbt_debug.png ├── project_level_dbt_dag.png ├── project_level_dbt_run.png ├── project_level_dbt_test.png ├── superset_daily_orders.png ├── superset_db_connection.png ├── great_expectations_dag_2.png ├── great_expectations_docs.png ├── model_level_dbt_compile.png ├── model_level_dbt_docs_task.png ├── model_level_dbt_validate.png ├── project_level_dbt_run_2.png ├── project_level_dbt_test_2.png ├── superset_dataset_creation.png ├── great_expectations_dag_targets.png ├── superset_jaffle_shop_dahsboard.png └── Screen Shot 2022-05-19 at 18.46.04.png ├── db.Dockerfile ├── airflow.Dockerfile ├── db ├── scripts │ ├── sql │ │ └── create_tables.sql │ └── create_db_insert_data.py └── data │ ├── jaffle_shop_customers.csv │ └── jaffle_shop_orders.csv └── superset ├── docker ├── pythonpath_dev │ ├── .gitignore │ └── superset_config_local.example ├── docker-frontend.sh ├── docker-entrypoint.sh ├── docker-ci.sh ├── .env-non-dev ├── frontend-mem-nag.sh ├── docker-bootstrap.sh ├── docker-init.sh └── README.md └── docker-compose-non-dev.yml /airflow/plugins/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | *.DS_Store 3 | *__pycache__ 4 | .env 5 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | dbt_packages/ 3 | logs/ 4 | -------------------------------------------------------------------------------- /airflow/logs/scheduler/latest: -------------------------------------------------------------------------------- 1 | /opt/airflow/logs/scheduler/2022-06-04 -------------------------------------------------------------------------------- /dbt/jaffle_shop/.user.yml: -------------------------------------------------------------------------------- 1 | id: 1b0091fc-1df3-4c2c-b858-c17490e77cbd 2 | -------------------------------------------------------------------------------- /great_expectations/.gitignore: -------------------------------------------------------------------------------- 1 | uncommitted/config_variables.yml 2 | uncommitted/.ipynb_checkpoints -------------------------------------------------------------------------------- /assets/dbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/dbt.png -------------------------------------------------------------------------------- /assets/diagram.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/diagram.jpg -------------------------------------------------------------------------------- /assets/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/diagram.png -------------------------------------------------------------------------------- /assets/dbt_docs_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/dbt_docs_1.png -------------------------------------------------------------------------------- /assets/dbt_docs_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/dbt_docs_2.png -------------------------------------------------------------------------------- /assets/postgresql.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/postgresql.png -------------------------------------------------------------------------------- /great_expectations/expectations/.ge_store_backend_id: -------------------------------------------------------------------------------- 1 | store_backend_id = 853ff521-99df-4e9e-bb0c-c9e52c1816d0 2 | -------------------------------------------------------------------------------- /assets/apache_airflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/apache_airflow.png -------------------------------------------------------------------------------- /assets/apache_superset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/apache_superset.png -------------------------------------------------------------------------------- /great_expectations/uncommitted/validations/.ge_store_backend_id: -------------------------------------------------------------------------------- 1 | store_backend_id = 024010e6-8e74-4266-ac58-15f622b1fb78 2 | -------------------------------------------------------------------------------- /assets/great-expectations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/great-expectations.png -------------------------------------------------------------------------------- /assets/model_level_dbt_run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/model_level_dbt_run.png -------------------------------------------------------------------------------- /assets/model_level_dbt_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/model_level_dbt_test.png -------------------------------------------------------------------------------- /assets/superset_daily_ltv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/superset_daily_ltv.png -------------------------------------------------------------------------------- /assets/great_expectations_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/great_expectations_dag.png -------------------------------------------------------------------------------- /assets/model_level_dbt_dag_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/model_level_dbt_dag_1.png -------------------------------------------------------------------------------- /assets/model_level_dbt_dag_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/model_level_dbt_dag_2.png -------------------------------------------------------------------------------- /assets/model_level_dbt_debug.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/model_level_dbt_debug.png -------------------------------------------------------------------------------- /assets/project_level_dbt_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/project_level_dbt_dag.png -------------------------------------------------------------------------------- /assets/project_level_dbt_run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/project_level_dbt_run.png -------------------------------------------------------------------------------- /assets/project_level_dbt_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/project_level_dbt_test.png -------------------------------------------------------------------------------- /assets/superset_daily_orders.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/superset_daily_orders.png -------------------------------------------------------------------------------- /assets/superset_db_connection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/superset_db_connection.png -------------------------------------------------------------------------------- /assets/great_expectations_dag_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/great_expectations_dag_2.png -------------------------------------------------------------------------------- /assets/great_expectations_docs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/great_expectations_docs.png -------------------------------------------------------------------------------- /assets/model_level_dbt_compile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/model_level_dbt_compile.png -------------------------------------------------------------------------------- /assets/model_level_dbt_docs_task.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/model_level_dbt_docs_task.png -------------------------------------------------------------------------------- /assets/model_level_dbt_validate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/model_level_dbt_validate.png -------------------------------------------------------------------------------- /assets/project_level_dbt_run_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/project_level_dbt_run_2.png -------------------------------------------------------------------------------- /assets/project_level_dbt_test_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/project_level_dbt_test_2.png -------------------------------------------------------------------------------- /assets/superset_dataset_creation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/superset_dataset_creation.png -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/graph.gpickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/dbt/jaffle_shop/target/graph.gpickle -------------------------------------------------------------------------------- /assets/great_expectations_dag_targets.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/great_expectations_dag_targets.png -------------------------------------------------------------------------------- /assets/superset_jaffle_shop_dahsboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/superset_jaffle_shop_dahsboard.png -------------------------------------------------------------------------------- /assets/Screen Shot 2022-05-19 at 18.46.04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/assets/Screen Shot 2022-05-19 at 18.46.04.png -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/partial_parse.msgpack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/dbt/jaffle_shop/target/partial_parse.msgpack -------------------------------------------------------------------------------- /dbt/jaffle_shop/macros/cents_to_dollars.sql: -------------------------------------------------------------------------------- 1 | {% macro cents_to_dollars(column_name, decimal_places=2) -%} 2 | round( 1.0 * {{ column_name }} / 100, {{ decimal_places }}) 3 | {%- endmacro %} -------------------------------------------------------------------------------- /db.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | 3 | COPY . /app 4 | 5 | WORKDIR /app 6 | 7 | RUN pip install pandas==1.2.3 8 | RUN pip install psycopg2-binary==2.9.1 9 | 10 | CMD python3 db/scripts/create_db_insert_data.py -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/images/favicon.ico -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/images/logo-long.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/images/logo-long.png -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/images/short-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/images/short-logo.png -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/stripe/src_stripe.yml/source_not_null_stripe_payment_id.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | select id 8 | from "raw"."stripe"."payment" 9 | where id is null 10 | 11 | 12 | -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/images/glossary_scroller.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/images/glossary_scroller.gif -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/images/iterative-dev-loop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/images/iterative-dev-loop.png -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-Bold.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-Bold.otf -------------------------------------------------------------------------------- /airflow.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/airflow:2.2.2 2 | 3 | RUN pip install dbt-postgres==1.1.0 4 | RUN pip install airflow-provider-great-expectations>=0.1.0 5 | RUN pip install great_expectations==0.15.5 6 | 7 | USER root 8 | RUN apt update 9 | RUN apt install -y git -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-Italic.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-Italic.otf -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-Light.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-Light.otf -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-Medium.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-Medium.otf -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-Regular.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-Regular.otf -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/src_jaffle_shop.yml/source_not_null_jaffle_shop_orders_id.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | select id 8 | from "raw"."jaffle_shop"."orders" 9 | where id is null 10 | 11 | 12 | -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-SemiBold.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-SemiBold.otf -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/src_jaffle_shop.yml/source_not_null_jaffle_shop_customers_id.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | select id 8 | from "raw"."jaffle_shop"."customers" 9 | where id is null 10 | 11 | 12 | -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-BoldItalic.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-BoldItalic.otf -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-LightItalic.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-LightItalic.otf -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-MediumItalic.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-MediumItalic.otf -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-SemiBoldItalic.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/fonts/HKGrotesk/HKGrotesk-SemiBoldItalic.otf -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/images/validation_failed_unexpected_values.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luchonaveiro/open-source-data-stack/HEAD/great_expectations/uncommitted/data_docs/local_site/static/images/validation_failed_unexpected_values.gif -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/not_null_stg_jaffle_shop__orders_order_id.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | select order_id 8 | from "raw"."dev"."stg_jaffle_shop__orders" 9 | where order_id is null 10 | 11 | 12 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/not_null_stg_jaffle_shop__customers_customer_id.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | select customer_id 8 | from "raw"."dev"."stg_jaffle_shop__customers" 9 | where customer_id is null 10 | 11 | 12 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/stripe/src_stripe.yml/source_unique_stripe_payment_id.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | select 6 | id as unique_field, 7 | count(*) as n_records 8 | 9 | from "raw"."stripe"."payment" 10 | where id is not null 11 | group by id 12 | having count(*) > 1 13 | 14 | 15 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/src_jaffle_shop.yml/source_unique_jaffle_shop_orders_id.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | select 6 | id as unique_field, 7 | count(*) as n_records 8 | 9 | from "raw"."jaffle_shop"."orders" 10 | where id is not null 11 | group by id 12 | having count(*) > 1 13 | 14 | 15 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/src_jaffle_shop.yml/source_unique_jaffle_shop_customers_id.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | select 6 | id as unique_field, 7 | count(*) as n_records 8 | 9 | from "raw"."jaffle_shop"."customers" 10 | where id is not null 11 | group by id 12 | having count(*) > 1 13 | 14 | 15 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/unique_stg_jaffle_shop__orders_order_id.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | select 6 | order_id as unique_field, 7 | count(*) as n_records 8 | 9 | from "raw"."dev"."stg_jaffle_shop__orders" 10 | where order_id is not null 11 | group by order_id 12 | having count(*) > 1 13 | 14 | 15 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/unique_stg_jaffle_shop__customers_customer_id.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | select 6 | customer_id as unique_field, 7 | count(*) as n_records 8 | 9 | from "raw"."dev"."stg_jaffle_shop__customers" 10 | where customer_id is not null 11 | group by customer_id 12 | having count(*) > 1 13 | 14 | 15 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/stripe/src_stripe.yml/source_not_null_stripe_payment_id.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | 11 | 12 | select id 13 | from "raw"."stripe"."payment" 14 | where id is null 15 | 16 | 17 | 18 | 19 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/tests/assert_positive_total_for_payments.sql: -------------------------------------------------------------------------------- 1 | WITH payments AS ( 2 | SELECT 3 | * 4 | FROM 5 | {{ ref ('stg_stripe__payments') }} 6 | ), 7 | test_data AS ( 8 | SELECT 9 | order_id, 10 | sum(payment_amount) AS total_amount 11 | FROM 12 | payments 13 | GROUP BY 14 | 1 15 | ) 16 | SELECT 17 | * 18 | FROM 19 | test_data 20 | WHERE 21 | total_amount < 0 22 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/src_jaffle_shop.yml/source_not_null_jaffle_shop_orders_id.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | 11 | 12 | select id 13 | from "raw"."jaffle_shop"."orders" 14 | where id is null 15 | 16 | 17 | 18 | 19 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/src_jaffle_shop.yml/source_not_null_jaffle_shop_customers_id.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | 11 | 12 | select id 13 | from "raw"."jaffle_shop"."customers" 14 | where id is null 15 | 16 | 17 | 18 | 19 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/tests/assert_positive_total_for_payments.sql: -------------------------------------------------------------------------------- 1 | WITH payments AS ( 2 | SELECT 3 | * 4 | FROM 5 | "raw"."dev"."stg_stripe__payments" 6 | ), 7 | test_data AS ( 8 | SELECT 9 | order_id, 10 | sum(payment_amount) AS total_amount 11 | FROM 12 | payments 13 | GROUP BY 14 | 1 15 | ) 16 | SELECT 17 | * 18 | FROM 19 | test_data 20 | WHERE 21 | total_amount < 0 -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/not_null_stg_jaffle_shop__orders_order_id.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | 11 | 12 | select order_id 13 | from "raw"."dev"."stg_jaffle_shop__orders" 14 | where order_id is null 15 | 16 | 17 | 18 | 19 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop__customers.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | source as ( 4 | 5 | select * from {{ source('jaffle_shop', 'customers') }} 6 | 7 | ), 8 | 9 | transformed as ( 10 | 11 | select 12 | 13 | id as customer_id, 14 | last_name as customer_last_name, 15 | first_name as customer_first_name, 16 | first_name || ' ' || last_name as full_name 17 | 18 | from source 19 | 20 | ) 21 | 22 | select * from transformed -------------------------------------------------------------------------------- /dbt/jaffle_shop/models/staging/stripe/stg_stripe__payments.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | source as ( 4 | 5 | select * from {{ source('stripe', 'payment') }} 6 | 7 | ), 8 | 9 | transformed as ( 10 | 11 | select 12 | 13 | id as payment_id, 14 | orderid as order_id, 15 | created as payment_created_at, 16 | status as payment_status, 17 | round(amount / 100.0, 2) as payment_amount 18 | 19 | from source 20 | 21 | ) 22 | 23 | select * from transformed 24 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/not_null_stg_jaffle_shop__customers_customer_id.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | 11 | 12 | select customer_id 13 | from "raw"."dev"."stg_jaffle_shop__customers" 14 | where customer_id is null 15 | 16 | 17 | 18 | 19 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop__customers.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | source as ( 4 | 5 | select * from "raw"."jaffle_shop"."customers" 6 | 7 | ), 8 | 9 | transformed as ( 10 | 11 | select 12 | 13 | id as customer_id, 14 | last_name as customer_last_name, 15 | first_name as customer_first_name, 16 | first_name || ' ' || last_name as full_name 17 | 18 | from source 19 | 20 | ) 21 | 22 | select * from transformed -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/stripe/stg_stripe__payments.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | source as ( 4 | 5 | select * from "raw"."stripe"."payment" 6 | 7 | ), 8 | 9 | transformed as ( 10 | 11 | select 12 | 13 | id as payment_id, 14 | orderid as order_id, 15 | created as payment_created_at, 16 | status as payment_status, 17 | round(amount / 100.0, 2) as payment_amount 18 | 19 | from source 20 | 21 | ) 22 | 23 | select * from transformed -------------------------------------------------------------------------------- /dbt/jaffle_shop/tests/assert_positive_total_for_payments_daily.sql: -------------------------------------------------------------------------------- 1 | WITH payments AS ( 2 | SELECT 3 | * 4 | FROM 5 | {{ ref ('stg_stripe__payments') }} 6 | WHERE 7 | payment_created_at = '{{ var ("date") }}' 8 | ), 9 | test_data AS ( 10 | SELECT 11 | order_id, 12 | sum(payment_amount) AS total_amount 13 | FROM 14 | payments 15 | GROUP BY 16 | 1 17 | ) 18 | SELECT 19 | * 20 | FROM 21 | test_data 22 | WHERE 23 | total_amount < 0 24 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/stripe/src_stripe.yml/source_unique_stripe_payment_id.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | select 11 | id as unique_field, 12 | count(*) as n_records 13 | 14 | from "raw"."stripe"."payment" 15 | where id is not null 16 | group by id 17 | having count(*) > 1 18 | 19 | 20 | 21 | 22 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/tests/assert_positive_total_for_payments_daily.sql: -------------------------------------------------------------------------------- 1 | WITH payments AS ( 2 | SELECT 3 | * 4 | FROM 5 | "raw"."dev"."stg_stripe__payments" 6 | WHERE 7 | payment_created_at = ' 2022-06-03 ' 8 | ), 9 | test_data AS ( 10 | SELECT 11 | order_id, 12 | sum(payment_amount) AS total_amount 13 | FROM 14 | payments 15 | GROUP BY 16 | 1 17 | ) 18 | SELECT 19 | * 20 | FROM 21 | test_data 22 | WHERE 23 | total_amount < 0 -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/src_jaffle_shop.yml/source_unique_jaffle_shop_orders_id.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | select 11 | id as unique_field, 12 | count(*) as n_records 13 | 14 | from "raw"."jaffle_shop"."orders" 15 | where id is not null 16 | group by id 17 | having count(*) > 1 18 | 19 | 20 | 21 | 22 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/src_jaffle_shop.yml/source_unique_jaffle_shop_customers_id.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | select 11 | id as unique_field, 12 | count(*) as n_records 13 | 14 | from "raw"."jaffle_shop"."customers" 15 | where id is not null 16 | group by id 17 | having count(*) > 1 18 | 19 | 20 | 21 | 22 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/models/staging/stripe/src_stripe.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: stripe 5 | database: raw 6 | schema: stripe 7 | tables: 8 | - name: payment 9 | columns: 10 | - name: id 11 | description: Primary key for payments. 12 | tests: 13 | - unique 14 | - not_null 15 | loaded_at_field: _batched_at 16 | freshness: 17 | warn_after: {count: 12, period: hour} 18 | error_after: {count: 24, period: hour} -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/accepted_values_stg_jaffle_sho_3ad15fa7d7f8de69cc0666c8351b51a7.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | with all_values as ( 6 | 7 | select 8 | status as value_field, 9 | count(*) as n_records 10 | 11 | from "raw"."dev"."stg_jaffle_shop__orders" 12 | group by status 13 | 14 | ) 15 | 16 | select * 17 | from all_values 18 | where value_field not in ( 19 | 'completed','shipped','returned','return_pending','placed' 20 | ) 21 | 22 | 23 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/accepted_values_stg_jaffle_sho_016d37c576e20b0d30ec2e634d2dac96.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | with all_values as ( 6 | 7 | select 8 | order_status as value_field, 9 | count(*) as n_records 10 | 11 | from "raw"."dev"."stg_jaffle_shop__orders" 12 | group by order_status 13 | 14 | ) 15 | 16 | select * 17 | from all_values 18 | where value_field not in ( 19 | 'completed','shipped','returned','return_pending','placed' 20 | ) 21 | 22 | 23 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/unique_stg_jaffle_shop__orders_order_id.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | select 11 | order_id as unique_field, 12 | count(*) as n_records 13 | 14 | from "raw"."dev"."stg_jaffle_shop__orders" 15 | where order_id is not null 16 | group by order_id 17 | having count(*) > 1 18 | 19 | 20 | 21 | 22 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/stripe/stg_stripe__payments.sql: -------------------------------------------------------------------------------- 1 | 2 | create view "raw"."dev"."stg_stripe__payments__dbt_tmp" as ( 3 | with 4 | 5 | source as ( 6 | 7 | select * from "raw"."stripe"."payment" 8 | 9 | ), 10 | 11 | transformed as ( 12 | 13 | select 14 | 15 | id as payment_id, 16 | orderid as order_id, 17 | created as payment_created_at, 18 | status as payment_status, 19 | round(amount / 100.0, 2) as payment_amount 20 | 21 | from source 22 | 23 | ) 24 | 25 | select * from transformed 26 | ); -------------------------------------------------------------------------------- /dbt/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop__orders.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | source as ( 4 | 5 | select * from {{ source('jaffle_shop', 'orders') }} 6 | 7 | ), 8 | 9 | transformed as ( 10 | 11 | select 12 | 13 | id as order_id, 14 | user_id as customer_id, 15 | order_date as order_placed_at, 16 | status as order_status, 17 | 18 | case 19 | when status not in ('returned','return_pending') 20 | then order_date 21 | end as valid_order_date 22 | 23 | from source 24 | 25 | ) 26 | 27 | select * from transformed -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/dbt_run/2021-12-23T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:20,691] {taskinstance.py:1029} INFO - Dependencies not met for , dependency 'Trigger Rule' FAILED: Task's trigger rule 'all_success' requires all upstream tasks to have succeeded, but found 1 non-success(es). upstream_tasks_state={'total': 1, 'successes': 0, 'skipped': 0, 'failed': 0, 'upstream_failed': 0, 'done': 0}, upstream_task_ids={'test'} 2 | [2022-05-15 01:30:20,787] {local_task_job.py:99} INFO - Task is not able to be run 3 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/dbt_run/2021-12-24T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:19,065] {taskinstance.py:1029} INFO - Dependencies not met for , dependency 'Trigger Rule' FAILED: Task's trigger rule 'all_success' requires all upstream tasks to have succeeded, but found 1 non-success(es). upstream_tasks_state={'total': 1, 'successes': 0, 'skipped': 0, 'failed': 0, 'upstream_failed': 0, 'done': 0}, upstream_task_ids={'test'} 2 | [2022-05-15 01:30:19,252] {local_task_job.py:99} INFO - Task is not able to be run 3 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/dbt_run/2021-12-25T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:15,921] {taskinstance.py:1029} INFO - Dependencies not met for , dependency 'Trigger Rule' FAILED: Task's trigger rule 'all_success' requires all upstream tasks to have succeeded, but found 1 non-success(es). upstream_tasks_state={'total': 1, 'successes': 0, 'skipped': 0, 'failed': 0, 'upstream_failed': 0, 'done': 0}, upstream_task_ids={'test'} 2 | [2022-05-15 01:30:15,976] {local_task_job.py:99} INFO - Task is not able to be run 3 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/dbt_run/2021-12-26T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:37,519] {taskinstance.py:1029} INFO - Dependencies not met for , dependency 'Trigger Rule' FAILED: Task's trigger rule 'all_success' requires all upstream tasks to have succeeded, but found 1 non-success(es). upstream_tasks_state={'total': 1, 'successes': 0, 'skipped': 0, 'failed': 0, 'upstream_failed': 0, 'done': 0}, upstream_task_ids={'test'} 2 | [2022-05-15 01:30:37,547] {local_task_job.py:99} INFO - Task is not able to be run 3 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/dbt_run/2021-12-27T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:37,618] {taskinstance.py:1029} INFO - Dependencies not met for , dependency 'Trigger Rule' FAILED: Task's trigger rule 'all_success' requires all upstream tasks to have succeeded, but found 1 non-success(es). upstream_tasks_state={'total': 1, 'successes': 0, 'skipped': 0, 'failed': 0, 'upstream_failed': 0, 'done': 0}, upstream_task_ids={'test'} 2 | [2022-05-15 01:30:37,997] {local_task_job.py:99} INFO - Task is not able to be run 3 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/dbt_run/2021-12-28T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:36,094] {taskinstance.py:1029} INFO - Dependencies not met for , dependency 'Trigger Rule' FAILED: Task's trigger rule 'all_success' requires all upstream tasks to have succeeded, but found 1 non-success(es). upstream_tasks_state={'total': 1, 'successes': 0, 'skipped': 0, 'failed': 0, 'upstream_failed': 0, 'done': 0}, upstream_task_ids={'test'} 2 | [2022-05-15 01:30:36,538] {local_task_job.py:99} INFO - Task is not able to be run 3 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/dbt_run/2021-12-30T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:38,975] {taskinstance.py:1029} INFO - Dependencies not met for , dependency 'Trigger Rule' FAILED: Task's trigger rule 'all_success' requires all upstream tasks to have succeeded, but found 1 non-success(es). upstream_tasks_state={'total': 1, 'successes': 0, 'skipped': 0, 'failed': 0, 'upstream_failed': 0, 'done': 0}, upstream_task_ids={'test'} 2 | [2022-05-15 01:30:39,025] {local_task_job.py:99} INFO - Task is not able to be run 3 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/dbt_run/2021-12-31T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:31:41,778] {taskinstance.py:1029} INFO - Dependencies not met for , dependency 'Trigger Rule' FAILED: Task's trigger rule 'all_success' requires all upstream tasks to have succeeded, but found 1 non-success(es). upstream_tasks_state={'total': 1, 'successes': 0, 'skipped': 0, 'failed': 0, 'upstream_failed': 0, 'done': 0}, upstream_task_ids={'test'} 2 | [2022-05-15 01:31:41,792] {local_task_job.py:99} INFO - Task is not able to be run 3 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop__customers.sql: -------------------------------------------------------------------------------- 1 | 2 | create view "raw"."dev"."stg_jaffle_shop__customers__dbt_tmp" as ( 3 | with 4 | 5 | source as ( 6 | 7 | select * from "raw"."jaffle_shop"."customers" 8 | 9 | ), 10 | 11 | transformed as ( 12 | 13 | select 14 | 15 | id as customer_id, 16 | last_name as customer_last_name, 17 | first_name as customer_first_name, 18 | first_name || ' ' || last_name as full_name 19 | 20 | from source 21 | 22 | ) 23 | 24 | select * from transformed 25 | ); -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/unique_stg_jaffle_shop__customers_customer_id.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | select 11 | customer_id as unique_field, 12 | count(*) as n_records 13 | 14 | from "raw"."dev"."stg_jaffle_shop__customers" 15 | where customer_id is not null 16 | group by customer_id 17 | having count(*) > 1 18 | 19 | 20 | 21 | 22 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop__orders.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | source as ( 4 | 5 | select * from "raw"."jaffle_shop"."orders" 6 | 7 | ), 8 | 9 | transformed as ( 10 | 11 | select 12 | 13 | id as order_id, 14 | user_id as customer_id, 15 | order_date as order_placed_at, 16 | status as order_status, 17 | 18 | case 19 | when status not in ('returned','return_pending') 20 | then order_date 21 | end as valid_order_date 22 | 23 | from source 24 | 25 | ) 26 | 27 | select * from transformed -------------------------------------------------------------------------------- /dbt/jaffle_shop/macros/grant_select.sql: -------------------------------------------------------------------------------- 1 | {% macro grant_select(schema=target.schema, role=target.role) %} 2 | 3 | {% set sql %} 4 | grant usage on schema {{ schema }} to role {{ role }}; 5 | grant select on all tables in schema {{ schema }} to role {{ role }}; 6 | grant select on all views in schema {{ schema }} to role {{ role }}; 7 | {% endset %} 8 | 9 | {{ log('Granting select on all tables and views in schema ' ~ target.schema ~ ' to role ' ~ role, info=True) }} 10 | {% do run_query(sql) %} 11 | {{ log('Privileges granted', info=True) }} 12 | 13 | {% endmacro %} 14 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/tests/assert_positive_total_for_payments.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | WITH payments AS ( 7 | SELECT 8 | * 9 | FROM 10 | "raw"."dev"."stg_stripe__payments" 11 | ), 12 | test_data AS ( 13 | SELECT 14 | order_id, 15 | sum(payment_amount) AS total_amount 16 | FROM 17 | payments 18 | GROUP BY 19 | 1 20 | ) 21 | SELECT 22 | * 23 | FROM 24 | test_data 25 | WHERE 26 | total_amount < 0 27 | 28 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/relationships_stg_jaffle_shop__69c2006accf24d593d7f144684ebb92c.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | with child as ( 6 | select customer_id as from_field 7 | from "raw"."dev"."stg_jaffle_shop__orders" 8 | where customer_id is not null 9 | ), 10 | 11 | parent as ( 12 | select customer_id as to_field 13 | from "raw"."dev"."stg_jaffle_shop__customers" 14 | ) 15 | 16 | select 17 | from_field 18 | 19 | from child 20 | left join parent 21 | on child.from_field = parent.to_field 22 | 23 | where parent.to_field is null 24 | 25 | 26 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.md: -------------------------------------------------------------------------------- 1 | {% docs order_status %} 2 | 3 | One of the following values: 4 | 5 | | status | definition | 6 | |----------------|--------------------------------------------------| 7 | | placed | Order placed, not yet shipped | 8 | | shipped | Order has been shipped, not yet been delivered | 9 | | completed | Order has been received by customers | 10 | | return pending | Customer indicated they want to return this item | 11 | | returned | Item has been returned | 12 | 13 | {% enddocs %} -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop__orders.sql: -------------------------------------------------------------------------------- 1 | 2 | create view "raw"."dev"."stg_jaffle_shop__orders__dbt_tmp" as ( 3 | with 4 | 5 | source as ( 6 | 7 | select * from "raw"."jaffle_shop"."orders" 8 | 9 | ), 10 | 11 | transformed as ( 12 | 13 | select 14 | 15 | id as order_id, 16 | user_id as customer_id, 17 | order_date as order_placed_at, 18 | status as order_status, 19 | 20 | case 21 | when status not in ('returned','return_pending') 22 | then order_date 23 | end as valid_order_date 24 | 25 | from source 26 | 27 | ) 28 | 29 | select * from transformed 30 | ); -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/tests/assert_positive_total_for_payments_daily.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | WITH payments AS ( 7 | SELECT 8 | * 9 | FROM 10 | "raw"."dev"."stg_stripe__payments" 11 | WHERE 12 | payment_created_at = ' 2022-06-03 ' 13 | ), 14 | test_data AS ( 15 | SELECT 16 | order_id, 17 | sum(payment_amount) AS total_amount 18 | FROM 19 | payments 20 | GROUP BY 21 | 1 22 | ) 23 | SELECT 24 | * 25 | FROM 26 | test_data 27 | WHERE 28 | total_amount < 0 29 | 30 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/accepted_values_stg_jaffle_sho_3ad15fa7d7f8de69cc0666c8351b51a7.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | with all_values as ( 11 | 12 | select 13 | status as value_field, 14 | count(*) as n_records 15 | 16 | from "raw"."dev"."stg_jaffle_shop__orders" 17 | group by status 18 | 19 | ) 20 | 21 | select * 22 | from all_values 23 | where value_field not in ( 24 | 'completed','shipped','returned','return_pending','placed' 25 | ) 26 | 27 | 28 | 29 | 30 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/accepted_values_stg_jaffle_sho_016d37c576e20b0d30ec2e634d2dac96.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | with all_values as ( 11 | 12 | select 13 | order_status as value_field, 14 | count(*) as n_records 15 | 16 | from "raw"."dev"."stg_jaffle_shop__orders" 17 | group by order_status 18 | 19 | ) 20 | 21 | select * 22 | from all_values 23 | where value_field not in ( 24 | 'completed','shipped','returned','return_pending','placed' 25 | ) 26 | 27 | 28 | 29 | 30 | ) dbt_internal_test -------------------------------------------------------------------------------- /dbt/jaffle_shop/profiles.yml: -------------------------------------------------------------------------------- 1 | config: 2 | use_colors: True 3 | jaffle_shop: 4 | outputs: 5 | dev: 6 | type: postgres 7 | threads: 1 8 | host: "{{ env_var('POSTGRES_HOST') }}" 9 | port: "{{ env_var('POSTGRES_PORT') | as_number }}" 10 | user: "{{ env_var('POSTGRES_USER') }}" 11 | pass: "{{ env_var('POSTGRES_PASSWORD') }}" 12 | dbname: "{{ env_var('POSTGRES_DB') }}" 13 | schema: dev 14 | prod: 15 | type: postgres 16 | threads: 1 17 | host: "{{ env_var('POSTGRES_HOST') }}" 18 | port: "{{ env_var('POSTGRES_PORT') | as_number }}" 19 | user: "{{ env_var('POSTGRES_USER') }}" 20 | pass: "{{ env_var('POSTGRES_PASSWORD') }}" 21 | dbname: "{{ env_var('POSTGRES_DB') }}" 22 | schema: analytics 23 | target: dev -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml/relationships_stg_jaffle_shop__69c2006accf24d593d7f144684ebb92c.sql: -------------------------------------------------------------------------------- 1 | select 2 | count(*) as failures, 3 | count(*) != 0 as should_warn, 4 | count(*) != 0 as should_error 5 | from ( 6 | 7 | 8 | 9 | 10 | with child as ( 11 | select customer_id as from_field 12 | from "raw"."dev"."stg_jaffle_shop__orders" 13 | where customer_id is not null 14 | ), 15 | 16 | parent as ( 17 | select customer_id as to_field 18 | from "raw"."dev"."stg_jaffle_shop__customers" 19 | ) 20 | 21 | select 22 | from_field 23 | 24 | from child 25 | left join parent 26 | on child.from_field = parent.to_field 27 | 28 | where parent.to_field is null 29 | 30 | 31 | 32 | 33 | ) dbt_internal_test -------------------------------------------------------------------------------- /great_expectations/plugins/custom_data_docs/styles/data_docs_custom_styles.css: -------------------------------------------------------------------------------- 1 | /*index page*/ 2 | .ge-index-page-site-name-title {} 3 | .ge-index-page-table-container {} 4 | .ge-index-page-table {} 5 | .ge-index-page-table-profiling-links-header {} 6 | .ge-index-page-table-expectations-links-header {} 7 | .ge-index-page-table-validations-links-header {} 8 | .ge-index-page-table-profiling-links-list {} 9 | .ge-index-page-table-profiling-links-item {} 10 | .ge-index-page-table-expectation-suite-link {} 11 | .ge-index-page-table-validation-links-list {} 12 | .ge-index-page-table-validation-links-item {} 13 | 14 | /*breadcrumbs*/ 15 | .ge-breadcrumbs {} 16 | .ge-breadcrumbs-item {} 17 | 18 | /*navigation sidebar*/ 19 | .ge-navigation-sidebar-container {} 20 | .ge-navigation-sidebar-content {} 21 | .ge-navigation-sidebar-title {} 22 | .ge-navigation-sidebar-link {} 23 | -------------------------------------------------------------------------------- /great_expectations/uncommitted/data_docs/local_site/static/styles/data_docs_custom_styles_template.css: -------------------------------------------------------------------------------- 1 | /*index page*/ 2 | .ge-index-page-site-name-title {} 3 | .ge-index-page-table-container {} 4 | .ge-index-page-table {} 5 | .ge-index-page-table-profiling-links-header {} 6 | .ge-index-page-table-expectations-links-header {} 7 | .ge-index-page-table-validations-links-header {} 8 | .ge-index-page-table-profiling-links-list {} 9 | .ge-index-page-table-profiling-links-item {} 10 | .ge-index-page-table-expectation-suite-link {} 11 | .ge-index-page-table-validation-links-list {} 12 | .ge-index-page-table-validation-links-item {} 13 | 14 | /*breadcrumbs*/ 15 | .ge-breadcrumbs {} 16 | .ge-breadcrumbs-item {} 17 | 18 | /*navigation sidebar*/ 19 | .ge-navigation-sidebar-container {} 20 | .ge-navigation-sidebar-content {} 21 | .ge-navigation-sidebar-title {} 22 | .ge-navigation-sidebar-link {} 23 | -------------------------------------------------------------------------------- /db/scripts/sql/create_tables.sql: -------------------------------------------------------------------------------- 1 | DROP SCHEMA IF EXISTS jaffle_shop CASCADE; 2 | 3 | CREATE SCHEMA IF NOT EXISTS jaffle_shop; 4 | 5 | DROP SCHEMA IF EXISTS stripe CASCADE; 6 | 7 | CREATE SCHEMA IF NOT EXISTS stripe; 8 | 9 | CREATE TABLE IF NOT EXISTS jaffle_shop.customers ( 10 | id int PRIMARY KEY, 11 | first_name varchar(255), 12 | last_name varchar(255) 13 | ); 14 | 15 | CREATE TABLE IF NOT EXISTS jaffle_shop.orders ( 16 | id int PRIMARY KEY, 17 | user_id int, 18 | order_date date, 19 | status varchar(255), 20 | _etl_loaded_at timestamp NOT NULL DEFAULT NOW() 21 | ); 22 | 23 | CREATE TABLE IF NOT EXISTS stripe.payment ( 24 | id int PRIMARY KEY, 25 | orderid int, 26 | paymentmethod varchar(255), 27 | status varchar(255), 28 | amount integer, 29 | created date, 30 | _batched_at timestamp NOT NULL DEFAULT NOW() 31 | ); 32 | 33 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/models/intermediate/int_orders.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | orders as ( 4 | 5 | select * from {{ ref('stg_jaffle_shop__orders') }} 6 | 7 | ), 8 | 9 | payments as ( 10 | 11 | select * from {{ ref('stg_stripe__payments') }} 12 | 13 | ), 14 | 15 | completed_payments as ( 16 | 17 | select 18 | order_id, 19 | max(payment_created_at) as payment_finalized_date, 20 | sum(payment_amount) as total_amount_paid 21 | from payments 22 | where payment_status <> 'fail' 23 | group by 1 24 | 25 | ), 26 | 27 | paid_orders as ( 28 | 29 | select 30 | orders.order_id, 31 | orders.customer_id, 32 | orders.order_placed_at, 33 | orders.order_status, 34 | completed_payments.total_amount_paid, 35 | completed_payments.payment_finalized_date 36 | from orders 37 | left join completed_payments on orders.order_id = completed_payments.order_id 38 | ) 39 | 40 | select * from paid_orders -------------------------------------------------------------------------------- /dbt/jaffle_shop/models/staging/jaffle_shop/src_jaffle_shop.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: jaffle_shop 5 | description: A clone of a Postgres application database. 6 | database: raw 7 | schema: jaffle_shop 8 | tables: 9 | - name: customers 10 | description: Raw customers data. 11 | columns: 12 | - name: id 13 | description: Primary key for customers. 14 | tests: 15 | - unique 16 | - not_null 17 | 18 | - name: orders 19 | description: Raw orders data. 20 | columns: 21 | - name: id 22 | description: Primary key for orders. 23 | tests: 24 | - unique 25 | - not_null 26 | loaded_at_field: _etl_loaded_at 27 | freshness: 28 | warn_after: {count: 12, period: hour} 29 | error_after: {count: 24, period: hour} -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/intermediate/int_orders.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | orders as ( 4 | 5 | select * from "raw"."dev"."stg_jaffle_shop__orders" 6 | 7 | ), 8 | 9 | payments as ( 10 | 11 | select * from "raw"."dev"."stg_stripe__payments" 12 | 13 | ), 14 | 15 | completed_payments as ( 16 | 17 | select 18 | order_id, 19 | max(payment_created_at) as payment_finalized_date, 20 | sum(payment_amount) as total_amount_paid 21 | from payments 22 | where payment_status <> 'fail' 23 | group by 1 24 | 25 | ), 26 | 27 | paid_orders as ( 28 | 29 | select 30 | orders.order_id, 31 | orders.customer_id, 32 | orders.order_placed_at, 33 | orders.order_status, 34 | completed_payments.total_amount_paid, 35 | completed_payments.payment_finalized_date 36 | from orders 37 | left join completed_payments on orders.order_id = completed_payments.order_id 38 | ) 39 | 40 | select * from paid_orders -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/intermediate/int_orders.sql: -------------------------------------------------------------------------------- 1 | 2 | create view "raw"."dev"."int_orders__dbt_tmp" as ( 3 | with 4 | 5 | orders as ( 6 | 7 | select * from "raw"."dev"."stg_jaffle_shop__orders" 8 | 9 | ), 10 | 11 | payments as ( 12 | 13 | select * from "raw"."dev"."stg_stripe__payments" 14 | 15 | ), 16 | 17 | completed_payments as ( 18 | 19 | select 20 | order_id, 21 | max(payment_created_at) as payment_finalized_date, 22 | sum(payment_amount) as total_amount_paid 23 | from payments 24 | where payment_status <> 'fail' 25 | group by 1 26 | 27 | ), 28 | 29 | paid_orders as ( 30 | 31 | select 32 | orders.order_id, 33 | orders.customer_id, 34 | orders.order_placed_at, 35 | orders.order_status, 36 | completed_payments.total_amount_paid, 37 | completed_payments.payment_finalized_date 38 | from orders 39 | left join completed_payments on orders.order_id = completed_payments.order_id 40 | ) 41 | 42 | select * from paid_orders 43 | ); -------------------------------------------------------------------------------- /superset/docker/pythonpath_dev/.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Ignore everything 19 | * 20 | # DON'T ignore the .gitignore 21 | !.gitignore 22 | !superset_config.py 23 | !superset_config_local.example 24 | -------------------------------------------------------------------------------- /great_expectations/checkpoints/sources/stripe__payment.yml: -------------------------------------------------------------------------------- 1 | name: stripe__payment 2 | config_version: 1.0 3 | template_name: 4 | module_name: great_expectations.checkpoint 5 | class_name: Checkpoint 6 | run_name_template: '%Y%m%d-%H%M%S-my-run-name-template' 7 | expectation_suite_name: 8 | batch_request: {} 9 | action_list: 10 | - name: store_validation_result 11 | action: 12 | class_name: StoreValidationResultAction 13 | - name: store_evaluation_params 14 | action: 15 | class_name: StoreEvaluationParametersAction 16 | - name: update_data_docs 17 | action: 18 | class_name: UpdateDataDocsAction 19 | site_names: [] 20 | evaluation_parameters: {} 21 | runtime_configuration: {} 22 | validations: 23 | - batch_request: 24 | datasource_name: jaffle_shop 25 | data_connector_name: default_inferred_data_connector_name 26 | data_asset_name: stripe.payment 27 | data_connector_query: 28 | index: -1 29 | expectation_suite_name: stripe.payment.warning 30 | profilers: [] 31 | ge_cloud_id: 32 | expectation_suite_ge_cloud_id: 33 | -------------------------------------------------------------------------------- /great_expectations/checkpoints/sources/jaffle_shop__orders.yml: -------------------------------------------------------------------------------- 1 | name: jaffle_shop__orders 2 | config_version: 1.0 3 | template_name: 4 | module_name: great_expectations.checkpoint 5 | class_name: Checkpoint 6 | run_name_template: '%Y%m%d-%H%M%S-my-run-name-template' 7 | expectation_suite_name: 8 | batch_request: {} 9 | action_list: 10 | - name: store_validation_result 11 | action: 12 | class_name: StoreValidationResultAction 13 | - name: store_evaluation_params 14 | action: 15 | class_name: StoreEvaluationParametersAction 16 | - name: update_data_docs 17 | action: 18 | class_name: UpdateDataDocsAction 19 | site_names: [] 20 | evaluation_parameters: {} 21 | runtime_configuration: {} 22 | validations: 23 | - batch_request: 24 | datasource_name: jaffle_shop 25 | data_connector_name: default_inferred_data_connector_name 26 | data_asset_name: jaffle_shop.orders 27 | data_connector_query: 28 | index: -1 29 | expectation_suite_name: jaffle_shop.orders.warning 30 | profilers: [] 31 | ge_cloud_id: 32 | expectation_suite_ge_cloud_id: 33 | -------------------------------------------------------------------------------- /great_expectations/checkpoints/sources/jaffle_shop__customers.yml: -------------------------------------------------------------------------------- 1 | name: jaffle_shop__customers 2 | config_version: 1.0 3 | template_name: 4 | module_name: great_expectations.checkpoint 5 | class_name: Checkpoint 6 | run_name_template: '%Y%m%d-%H%M%S-my-run-name-template' 7 | expectation_suite_name: 8 | batch_request: {} 9 | action_list: 10 | - name: store_validation_result 11 | action: 12 | class_name: StoreValidationResultAction 13 | - name: store_evaluation_params 14 | action: 15 | class_name: StoreEvaluationParametersAction 16 | - name: update_data_docs 17 | action: 18 | class_name: UpdateDataDocsAction 19 | site_names: [] 20 | evaluation_parameters: {} 21 | runtime_configuration: {} 22 | validations: 23 | - batch_request: 24 | datasource_name: jaffle_shop 25 | data_connector_name: default_inferred_data_connector_name 26 | data_asset_name: jaffle_shop.customers 27 | data_connector_query: 28 | index: -1 29 | expectation_suite_name: jaffle_shop.customers.warning 30 | profilers: [] 31 | ge_cloud_id: 32 | expectation_suite_ge_cloud_id: 33 | -------------------------------------------------------------------------------- /superset/docker/docker-frontend.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | set -e 19 | 20 | cd /app/superset-frontend 21 | npm install -g npm@7 22 | npm install -f --no-optional --global webpack webpack-cli 23 | npm install -f --no-optional 24 | 25 | echo "Running frontend" 26 | npm run dev 27 | -------------------------------------------------------------------------------- /great_expectations/checkpoints/targets/dev__fct_customer_orders.yml: -------------------------------------------------------------------------------- 1 | name: targets.dev__fct_customer_orders 2 | config_version: 1.0 3 | template_name: 4 | module_name: great_expectations.checkpoint 5 | class_name: Checkpoint 6 | run_name_template: '%Y%m%d-%H%M%S-my-run-name-template' 7 | expectation_suite_name: 8 | batch_request: {} 9 | action_list: 10 | - name: store_validation_result 11 | action: 12 | class_name: StoreValidationResultAction 13 | - name: store_evaluation_params 14 | action: 15 | class_name: StoreEvaluationParametersAction 16 | - name: update_data_docs 17 | action: 18 | class_name: UpdateDataDocsAction 19 | site_names: [] 20 | evaluation_parameters: {} 21 | runtime_configuration: {} 22 | validations: 23 | - batch_request: 24 | datasource_name: jaffle_shop 25 | data_connector_name: default_inferred_data_connector_name 26 | data_asset_name: dev.fct_customer_orders 27 | data_connector_query: 28 | index: -1 29 | expectation_suite_name: dev.fct_customer_orders.warning 30 | profilers: [] 31 | ge_cloud_id: 32 | expectation_suite_ge_cloud_id: 33 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/models/staging/jaffle_shop/stg_jaffle_shop.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_jaffle_shop__customers 5 | description: Staged customer data from our jaffle shop app. 6 | columns: 7 | - name: customer_id 8 | description: The primary key for customers. 9 | tests: 10 | - unique 11 | - not_null 12 | 13 | - name: stg_jaffle_shop__orders 14 | description: Staged order data from our jaffle shop app. 15 | columns: 16 | - name: order_id 17 | description: Primary key for orders. 18 | tests: 19 | - unique 20 | - not_null 21 | - name: order_status 22 | description: "{{ doc('order_status') }}" 23 | tests: 24 | - accepted_values: 25 | values: 26 | - completed 27 | - shipped 28 | - returned 29 | - return_pending 30 | - placed 31 | - name: customer_id 32 | description: Foreign key to stg_customers.customer_id. 33 | tests: 34 | - relationships: 35 | to: ref('stg_jaffle_shop__customers') 36 | field: customer_id -------------------------------------------------------------------------------- /dbt/jaffle_shop/macros/clean_stale_models.sql: -------------------------------------------------------------------------------- 1 | {% macro clean_stale_models(database=target.database, schema=target.schema, days=7, dry_run=True) %} 2 | 3 | {% set get_drop_commands_query %} 4 | select 5 | case 6 | when table_type = 'VIEW' 7 | then table_type 8 | else 9 | 'TABLE' 10 | end as drop_type, 11 | 'DROP ' || drop_type || ' {{ database | upper }}.' || table_schema || '.' || table_name || ';' 12 | from {{ database }}.information_schema.tables 13 | where table_schema = upper('{{ schema }}') 14 | and last_altered <= current_date - {{ days }} 15 | {% endset %} 16 | 17 | {{ log('\nGenerating cleanup queries...\n', info=True) }} 18 | {% set drop_queries = run_query(get_drop_commands_query).columns[1].values() %} 19 | 20 | {% for query in drop_queries %} 21 | {% if dry_run %} 22 | {{ log(query, info=True) }} 23 | {% else %} 24 | {{ log('Dropping object with command: ' ~ query, info=True) }} 25 | {% do run_query(query) %} 26 | {% endif %} 27 | {% endfor %} 28 | 29 | {% endmacro %} -------------------------------------------------------------------------------- /airflow/dags/project_level_dbt_dag.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | from airflow import DAG 4 | from airflow.operators.bash_operator import BashOperator 5 | from airflow.utils.dates import datetime 6 | from airflow.utils.dates import timedelta 7 | 8 | DAG_ID = "project_level_dbt_dag" 9 | DAG_OWNER = "luciano.naveiro" 10 | 11 | DBT_PROJECT_PATH = "/opt/dbt/jaffle_shop" 12 | 13 | default_args = { 14 | "owner": DAG_OWNER, 15 | "retries": 3, 16 | "retry_delay": timedelta(minutes=2), 17 | } 18 | 19 | 20 | with DAG( 21 | DAG_ID, 22 | start_date=datetime(2021, 12, 23), 23 | description="An Airflow DAG to invoke simple dbt commands", 24 | schedule_interval=timedelta(days=1), 25 | default_args=default_args, 26 | catchup=False, 27 | ) as dag: 28 | 29 | dbt_run = BashOperator( 30 | task_id="dbt_run", 31 | bash_command=f""" 32 | dbt run --profiles-dir {DBT_PROJECT_PATH} --project-dir {DBT_PROJECT_PATH} 33 | """, 34 | ) 35 | 36 | dbt_test = BashOperator( 37 | task_id="dbt_test", 38 | bash_command=f""" 39 | dbt test --profiles-dir {DBT_PROJECT_PATH} --project-dir {DBT_PROJECT_PATH} --vars '{{"date": " {{{{ ds }}}} " }}' 40 | """, 41 | ) 42 | 43 | dbt_run >> dbt_test 44 | -------------------------------------------------------------------------------- /superset/docker/pythonpath_dev/superset_config_local.example: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # 19 | # This is an example "local" configuration file. In order to set/override config 20 | # options that ONLY apply to your local environment, simply copy/rename this file 21 | # to docker/pythonpath/superset_config_docker.py 22 | # It ends up being imported by docker/superset_config.py which is loaded by 23 | # superset/config.py 24 | # 25 | 26 | SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://pguser:pgpwd@some.host/superset" 27 | SQLALCHEMY_ECHO = True 28 | -------------------------------------------------------------------------------- /superset/docker/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | set -eo pipefail 19 | 20 | if [ "${#}" -ne 0 ]; then 21 | exec "${@}" 22 | else 23 | gunicorn \ 24 | --bind "0.0.0.0:${SUPERSET_PORT}" \ 25 | --access-logfile '-' \ 26 | --error-logfile '-' \ 27 | --workers 1 \ 28 | --worker-class gthread \ 29 | --threads 20 \ 30 | --timeout ${GUNICORN_TIMEOUT:-60} \ 31 | --limit-request-line 0 \ 32 | --limit-request-field_size 0 \ 33 | "${FLASK_APP}" 34 | fi 35 | -------------------------------------------------------------------------------- /superset/docker/docker-ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | /app/docker/docker-init.sh 19 | 20 | # TODO: copy config overrides from ENV vars 21 | 22 | # TODO: run celery in detached state 23 | 24 | # start up the web server 25 | gunicorn \ 26 | --bind "0.0.0.0:${SUPERSET_PORT}" \ 27 | --access-logfile '-' \ 28 | --error-logfile '-' \ 29 | --workers 1 \ 30 | --worker-class gthread \ 31 | --threads 8 \ 32 | --timeout 60 \ 33 | --limit-request-line 0 \ 34 | --limit-request-field_size 0 \ 35 | "${FLASK_APP}" 36 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | # Name your project! Project names should contain only lowercase characters 3 | # and underscores. A good package name should reflect your organization's 4 | # name or the intended use of these models 5 | name: 'jaffle_shop' 6 | version: '1.0.0' 7 | config-version: 2 8 | 9 | # This setting configures which "profile" dbt uses for this project. 10 | profile: 'jaffle_shop' 11 | 12 | # These configurations specify where dbt should look for different types of files. 13 | # The `model-paths` config, for example, states that models in this project can be 14 | # found in the "models/" directory. You probably won't need to change these! 15 | model-paths: ["models"] 16 | analysis-paths: ["analyses"] 17 | test-paths: ["tests"] 18 | seed-paths: ["seeds"] 19 | macro-paths: ["macros"] 20 | snapshot-paths: ["snapshots"] 21 | 22 | target-path: "target" # directory which will store compiled SQL files 23 | clean-targets: # directories to be removed by `dbt clean` 24 | - "target" 25 | - "dbt_packages" 26 | 27 | 28 | # Configuring models 29 | # Full documentation: https://docs.getdbt.com/docs/configuring-models 30 | 31 | # In this example config, we tell dbt to build all models in the example/ directory 32 | # as tables. These settings can be overridden in the individual model files 33 | # using the `{{ config(...) }}` macro. 34 | models: 35 | jaffle_shop: 36 | staging: 37 | +materialized: view 38 | marts: 39 | core: 40 | +materialized: table 41 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/models/marts/core/fct_customer_orders.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | customers as ( 4 | 5 | select * from {{ ref('stg_jaffle_shop__customers') }} 6 | 7 | ), 8 | 9 | paid_orders as ( 10 | 11 | select * from {{ ref('int_orders') }} 12 | 13 | ), 14 | 15 | final as ( 16 | 17 | select 18 | paid_orders.order_id, 19 | paid_orders.customer_id, 20 | paid_orders.order_placed_at, 21 | paid_orders.order_status, 22 | paid_orders.total_amount_paid, 23 | paid_orders.payment_finalized_date, 24 | customers.customer_first_name, 25 | customers.customer_last_name, 26 | 27 | -- sales transaction sequence 28 | row_number() over (order by paid_orders.order_id) as transaction_seq, 29 | 30 | -- customer sales sequence 31 | row_number() over (partition by paid_orders.customer_id order by paid_orders.order_id) as customer_sales_seq, 32 | 33 | -- new vs returning customer 34 | case 35 | when ( 36 | rank() over ( 37 | partition by paid_orders.customer_id 38 | order by paid_orders.order_placed_at, paid_orders.order_id 39 | ) = 1 40 | ) then 'new' 41 | else 'return' end as nvsr, 42 | 43 | -- customer lifetime value 44 | sum(paid_orders.total_amount_paid) over ( 45 | partition by paid_orders.customer_id 46 | order by paid_orders.order_placed_at 47 | ) as customer_lifetime_value, 48 | 49 | -- first day of sale 50 | first_value(order_placed_at) over ( 51 | partition by paid_orders.customer_id 52 | order by paid_orders.order_placed_at 53 | ) as fdos 54 | from paid_orders 55 | left join customers on paid_orders.customer_id = customers.customer_id 56 | ) 57 | 58 | select * from final -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/marts/core/fct_customer_orders.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | customers as ( 4 | 5 | select * from "raw"."dev"."stg_jaffle_shop__customers" 6 | 7 | ), 8 | 9 | paid_orders as ( 10 | 11 | select * from "raw"."dev"."int_orders" 12 | 13 | ), 14 | 15 | final as ( 16 | 17 | select 18 | paid_orders.order_id, 19 | paid_orders.customer_id, 20 | paid_orders.order_placed_at, 21 | paid_orders.order_status, 22 | paid_orders.total_amount_paid, 23 | paid_orders.payment_finalized_date, 24 | customers.customer_first_name, 25 | customers.customer_last_name, 26 | 27 | -- sales transaction sequence 28 | row_number() over (order by paid_orders.order_id) as transaction_seq, 29 | 30 | -- customer sales sequence 31 | row_number() over (partition by paid_orders.customer_id order by paid_orders.order_id) as customer_sales_seq, 32 | 33 | -- new vs returning customer 34 | case 35 | when ( 36 | rank() over ( 37 | partition by paid_orders.customer_id 38 | order by paid_orders.order_placed_at, paid_orders.order_id 39 | ) = 1 40 | ) then 'new' 41 | else 'return' end as nvsr, 42 | 43 | -- customer lifetime value 44 | sum(paid_orders.total_amount_paid) over ( 45 | partition by paid_orders.customer_id 46 | order by paid_orders.order_placed_at 47 | ) as customer_lifetime_value, 48 | 49 | -- first day of sale 50 | first_value(order_placed_at) over ( 51 | partition by paid_orders.customer_id 52 | order by paid_orders.order_placed_at 53 | ) as fdos 54 | from paid_orders 55 | left join customers on paid_orders.customer_id = customers.customer_id 56 | ) 57 | 58 | select * from final -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/marts/core/fct_customer_orders_new.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | customers as ( 4 | 5 | select * from "raw"."dev"."stg_jaffle_shop__customers" 6 | 7 | ), 8 | 9 | paid_orders as ( 10 | 11 | select * from "raw"."dev"."int_orders" 12 | 13 | ), 14 | 15 | final as ( 16 | 17 | select 18 | paid_orders.order_id, 19 | paid_orders.customer_id, 20 | paid_orders.order_placed_at, 21 | paid_orders.order_status, 22 | paid_orders.total_amount_paid, 23 | paid_orders.payment_finalized_date, 24 | customers.customer_first_name, 25 | customers.customer_last_name, 26 | 27 | -- sales transaction sequence 28 | row_number() over (order by paid_orders.order_id) as transaction_seq, 29 | 30 | -- customer sales sequence 31 | row_number() over (partition by paid_orders.customer_id order by paid_orders.order_id) as customer_sales_seq, 32 | 33 | -- new vs returning customer 34 | case 35 | when ( 36 | rank() over ( 37 | partition by paid_orders.customer_id 38 | order by paid_orders.order_placed_at, paid_orders.order_id 39 | ) = 1 40 | ) then 'new' 41 | else 'return' end as nvsr, 42 | 43 | -- customer lifetime value 44 | sum(paid_orders.total_amount_paid) over ( 45 | partition by paid_orders.customer_id 46 | order by paid_orders.order_placed_at 47 | ) as customer_lifetime_value, 48 | 49 | -- first day of sale 50 | first_value(order_placed_at) over ( 51 | partition by paid_orders.customer_id 52 | order by paid_orders.order_placed_at 53 | ) as fdos 54 | from paid_orders 55 | left join customers on paid_orders.customer_id = customers.customer_id 56 | ) 57 | 58 | select * from final -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/compiled/jaffle_shop/models/marts/core/fct_customer_orders_new2.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | customers as ( 4 | 5 | select * from "raw"."dev"."stg_jaffle_shop__customers" 6 | 7 | ), 8 | 9 | paid_orders as ( 10 | 11 | select * from "raw"."dev"."int_orders" 12 | 13 | ), 14 | 15 | final as ( 16 | 17 | select 18 | paid_orders.order_id, 19 | paid_orders.customer_id, 20 | paid_orders.order_placed_at, 21 | paid_orders.order_status, 22 | paid_orders.total_amount_paid, 23 | paid_orders.payment_finalized_date, 24 | customers.customer_first_name, 25 | customers.customer_last_name, 26 | 27 | -- sales transaction sequence 28 | row_number() over (order by paid_orders.order_id) as transaction_seq, 29 | 30 | -- customer sales sequence 31 | row_number() over (partition by paid_orders.customer_id order by paid_orders.order_id) as customer_sales_seq, 32 | 33 | -- new vs returning customer 34 | case 35 | when ( 36 | rank() over ( 37 | partition by paid_orders.customer_id 38 | order by paid_orders.order_placed_at, paid_orders.order_id 39 | ) = 1 40 | ) then 'new' 41 | else 'return' end as nvsr, 42 | 43 | -- customer lifetime value 44 | sum(paid_orders.total_amount_paid) over ( 45 | partition by paid_orders.customer_id 46 | order by paid_orders.order_placed_at 47 | ) as customer_lifetime_value, 48 | 49 | -- first day of sale 50 | first_value(order_placed_at) over ( 51 | partition by paid_orders.customer_id 52 | order by paid_orders.order_placed_at 53 | ) as fdos 54 | from paid_orders 55 | left join customers on paid_orders.customer_id = customers.customer_id 56 | ) 57 | 58 | select * from final -------------------------------------------------------------------------------- /superset/docker/.env-non-dev: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | COMPOSE_PROJECT_NAME=superset 18 | 19 | # database configurations (do not modify) 20 | DATABASE_DB=superset 21 | DATABASE_HOST=db 22 | DATABASE_PASSWORD=superset 23 | DATABASE_USER=superset 24 | 25 | # database engine specific environment variables 26 | # change the below if you prefers another database engine 27 | DATABASE_PORT=5432 28 | DATABASE_DIALECT=postgresql 29 | POSTGRES_DB=superset 30 | POSTGRES_USER=superset 31 | POSTGRES_PASSWORD=superset 32 | #MYSQL_DATABASE=superset 33 | #MYSQL_USER=superset 34 | #MYSQL_PASSWORD=superset 35 | #MYSQL_RANDOM_ROOT_PASSWORD=yes 36 | 37 | # Add the mapped in /app/pythonpath_docker which allows devs to override stuff 38 | PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev 39 | REDIS_HOST=redis 40 | REDIS_PORT=6379 41 | 42 | FLASK_ENV=production 43 | SUPERSET_ENV=production 44 | SUPERSET_LOAD_EXAMPLES=no 45 | CYPRESS_CONFIG=false 46 | SUPERSET_PORT=8088 47 | -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/marts/core/fct_customer_orders.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | create table "raw"."dev"."fct_customer_orders__dbt_tmp" 4 | as ( 5 | with 6 | 7 | customers as ( 8 | 9 | select * from "raw"."dev"."stg_jaffle_shop__customers" 10 | 11 | ), 12 | 13 | paid_orders as ( 14 | 15 | select * from "raw"."dev"."int_orders" 16 | 17 | ), 18 | 19 | final as ( 20 | 21 | select 22 | paid_orders.order_id, 23 | paid_orders.customer_id, 24 | paid_orders.order_placed_at, 25 | paid_orders.order_status, 26 | paid_orders.total_amount_paid, 27 | paid_orders.payment_finalized_date, 28 | customers.customer_first_name, 29 | customers.customer_last_name, 30 | 31 | -- sales transaction sequence 32 | row_number() over (order by paid_orders.order_id) as transaction_seq, 33 | 34 | -- customer sales sequence 35 | row_number() over (partition by paid_orders.customer_id order by paid_orders.order_id) as customer_sales_seq, 36 | 37 | -- new vs returning customer 38 | case 39 | when ( 40 | rank() over ( 41 | partition by paid_orders.customer_id 42 | order by paid_orders.order_placed_at, paid_orders.order_id 43 | ) = 1 44 | ) then 'new' 45 | else 'return' end as nvsr, 46 | 47 | -- customer lifetime value 48 | sum(paid_orders.total_amount_paid) over ( 49 | partition by paid_orders.customer_id 50 | order by paid_orders.order_placed_at 51 | ) as customer_lifetime_value, 52 | 53 | -- first day of sale 54 | first_value(order_placed_at) over ( 55 | partition by paid_orders.customer_id 56 | order by paid_orders.order_placed_at 57 | ) as fdos 58 | from paid_orders 59 | left join customers on paid_orders.customer_id = customers.customer_id 60 | ) 61 | 62 | select * from final 63 | ); -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/marts/core/fct_customer_orders_new.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | create table "raw"."dev"."fct_customer_orders_new__dbt_tmp" 4 | as ( 5 | with 6 | 7 | customers as ( 8 | 9 | select * from "raw"."dev"."stg_jaffle_shop__customers" 10 | 11 | ), 12 | 13 | paid_orders as ( 14 | 15 | select * from "raw"."dev"."int_orders" 16 | 17 | ), 18 | 19 | final as ( 20 | 21 | select 22 | paid_orders.order_id, 23 | paid_orders.customer_id, 24 | paid_orders.order_placed_at, 25 | paid_orders.order_status, 26 | paid_orders.total_amount_paid, 27 | paid_orders.payment_finalized_date, 28 | customers.customer_first_name, 29 | customers.customer_last_name, 30 | 31 | -- sales transaction sequence 32 | row_number() over (order by paid_orders.order_id) as transaction_seq, 33 | 34 | -- customer sales sequence 35 | row_number() over (partition by paid_orders.customer_id order by paid_orders.order_id) as customer_sales_seq, 36 | 37 | -- new vs returning customer 38 | case 39 | when ( 40 | rank() over ( 41 | partition by paid_orders.customer_id 42 | order by paid_orders.order_placed_at, paid_orders.order_id 43 | ) = 1 44 | ) then 'new' 45 | else 'return' end as nvsr, 46 | 47 | -- customer lifetime value 48 | sum(paid_orders.total_amount_paid) over ( 49 | partition by paid_orders.customer_id 50 | order by paid_orders.order_placed_at 51 | ) as customer_lifetime_value, 52 | 53 | -- first day of sale 54 | first_value(order_placed_at) over ( 55 | partition by paid_orders.customer_id 56 | order by paid_orders.order_placed_at 57 | ) as fdos 58 | from paid_orders 59 | left join customers on paid_orders.customer_id = customers.customer_id 60 | ) 61 | 62 | select * from final 63 | ); -------------------------------------------------------------------------------- /dbt/jaffle_shop/target/run/jaffle_shop/models/marts/core/fct_customer_orders_new2.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | create table "raw"."dev"."fct_customer_orders_new2__dbt_tmp" 4 | as ( 5 | with 6 | 7 | customers as ( 8 | 9 | select * from "raw"."dev"."stg_jaffle_shop__customers" 10 | 11 | ), 12 | 13 | paid_orders as ( 14 | 15 | select * from "raw"."dev"."int_orders" 16 | 17 | ), 18 | 19 | final as ( 20 | 21 | select 22 | paid_orders.order_id, 23 | paid_orders.customer_id, 24 | paid_orders.order_placed_at, 25 | paid_orders.order_status, 26 | paid_orders.total_amount_paid, 27 | paid_orders.payment_finalized_date, 28 | customers.customer_first_name, 29 | customers.customer_last_name, 30 | 31 | -- sales transaction sequence 32 | row_number() over (order by paid_orders.order_id) as transaction_seq, 33 | 34 | -- customer sales sequence 35 | row_number() over (partition by paid_orders.customer_id order by paid_orders.order_id) as customer_sales_seq, 36 | 37 | -- new vs returning customer 38 | case 39 | when ( 40 | rank() over ( 41 | partition by paid_orders.customer_id 42 | order by paid_orders.order_placed_at, paid_orders.order_id 43 | ) = 1 44 | ) then 'new' 45 | else 'return' end as nvsr, 46 | 47 | -- customer lifetime value 48 | sum(paid_orders.total_amount_paid) over ( 49 | partition by paid_orders.customer_id 50 | order by paid_orders.order_placed_at 51 | ) as customer_lifetime_value, 52 | 53 | -- first day of sale 54 | first_value(order_placed_at) over ( 55 | partition by paid_orders.customer_id 56 | order by paid_orders.order_placed_at 57 | ) as fdos 58 | from paid_orders 59 | left join customers on paid_orders.customer_id = customers.customer_id 60 | ) 61 | 62 | select * from final 63 | ); -------------------------------------------------------------------------------- /db/data/jaffle_shop_customers.csv: -------------------------------------------------------------------------------- 1 | ID,FIRST_NAME,LAST_NAME 2 | 1,Michael,P. 3 | 2,Shawn,M. 4 | 3,Kathleen,P. 5 | 4,Jimmy,C. 6 | 5,Katherine,R. 7 | 6,Sarah,R. 8 | 7,Martin,M. 9 | 8,Frank,R. 10 | 9,Jennifer,F. 11 | 10,Henry,W. 12 | 11,Fred,S. 13 | 12,Amy,D. 14 | 13,Kathleen,M. 15 | 14,Steve,F. 16 | 15,Teresa,H. 17 | 16,Amanda,H. 18 | 17,Kimberly,R. 19 | 18,Johnny,K. 20 | 19,Virginia,F. 21 | 20,Anna,A. 22 | 21,Willie,H. 23 | 22,Sean,H. 24 | 23,Mildred,A. 25 | 24,David,G. 26 | 25,Victor,H. 27 | 26,Aaron,R. 28 | 27,Benjamin,B. 29 | 28,Lisa,W. 30 | 29,Benjamin,K. 31 | 30,Christina,W. 32 | 31,Jane,G. 33 | 32,Thomas,O. 34 | 33,Katherine,M. 35 | 34,Jennifer,S. 36 | 35,Sara,T. 37 | 36,Harold,O. 38 | 37,Shirley,J. 39 | 38,Dennis,J. 40 | 39,Louise,W. 41 | 40,Maria,A. 42 | 41,Gloria,C. 43 | 42,Diana,S. 44 | 43,Kelly,N. 45 | 44,Jane,R. 46 | 45,Scott,B. 47 | 46,Norma,C. 48 | 47,Marie,P. 49 | 48,Lillian,C. 50 | 49,Judy,N. 51 | 50,Billy,L. 52 | 51,Howard,R. 53 | 52,Laura,F. 54 | 53,Anne,B. 55 | 54,Rose,M. 56 | 55,Nicholas,R. 57 | 56,Joshua,K. 58 | 57,Paul,W. 59 | 58,Kathryn,K. 60 | 59,Adam,A. 61 | 60,Norma,W. 62 | 61,Timothy,R. 63 | 62,Elizabeth,P. 64 | 63,Edward,G. 65 | 64,David,C. 66 | 65,Brenda,W. 67 | 66,Adam,W. 68 | 67,Michael,H. 69 | 68,Jesse,E. 70 | 69,Janet,P. 71 | 70,Helen,F. 72 | 71,Gerald,C. 73 | 72,Kathryn,O. 74 | 73,Alan,B. 75 | 74,Harry,A. 76 | 75,Andrea,H. 77 | 76,Barbara,W. 78 | 77,Anne,W. 79 | 78,Harry,H. 80 | 79,Jack,R. 81 | 80,Phillip,H. 82 | 81,Shirley,H. 83 | 82,Arthur,D. 84 | 83,Virginia,R. 85 | 84,Christina,R. 86 | 85,Theresa,M. 87 | 86,Jason,C. 88 | 87,Phillip,B. 89 | 88,Adam,T. 90 | 89,Margaret,J. 91 | 90,Paul,P. 92 | 91,Todd,W. 93 | 92,Willie,O. 94 | 93,Frances,R. 95 | 94,Gregory,H. 96 | 95,Lisa,P. 97 | 96,Jacqueline,A. 98 | 97,Shirley,D. 99 | 98,Nicole,M. 100 | 99,Mary,G. 101 | 100,Jean,M. -------------------------------------------------------------------------------- /superset/docker/frontend-mem-nag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | set -e 20 | 21 | # We need at least 3GB of free mem... 22 | MIN_MEM_FREE_GB=3 23 | MIN_MEM_FREE_KB=$(($MIN_MEM_FREE_GB*1000000)) 24 | 25 | echo_mem_warn() { 26 | MEM_FREE_KB=$(awk '/MemFree/ { printf "%s \n", $2 }' /proc/meminfo) 27 | MEM_FREE_GB=$(awk '/MemFree/ { printf "%s \n", $2/1024/1024 }' /proc/meminfo) 28 | 29 | if [[ "${MEM_FREE_KB}" -lt "${MIN_MEM_FREE_KB}" ]]; then 30 | cat < 2 | [2022-05-15 01:31:41,946] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:31:41,949] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:31:41,953] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:31:41,955] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:31:42,013] {taskinstance.py:1262} INFO - Executing on 2021-12-31 00:00:00+00:00 9 | [2022-05-15 01:31:42,039] {standard_task_runner.py:52} INFO - Started process 2644 to run task 10 | [2022-05-15 01:31:42,082] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2021-12-31T00:00:00+00:00', '--job-id', '56', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmp5uol6brv', '--error-file', '/tmp/tmp8p79a2wr'] 11 | [2022-05-15 01:31:42,084] {standard_task_runner.py:77} INFO - Job 56: Subtask test 12 | [2022-05-15 01:31:42,187] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:31:42,375] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2021-12-31T00:00:00+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2021-12-31T00:00:00+00:00 19 | [2022-05-15 01:31:42,386] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:31:42,390] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:31:42,409] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:31:42,412] {subprocess.py:89} INFO - /tmp/***tmp26zcmu4_ 24 | [2022-05-15 01:31:42,419] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:31:42,468] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20211231T000000, start_date=20220515T013141, end_date=20220515T013142 26 | [2022-05-15 01:31:42,539] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:31:42,612] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 28 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-01-01T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:31:41,742] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:31:41,770] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:31:41,771] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:31:41,772] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:31:41,773] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:31:41,877] {taskinstance.py:1262} INFO - Executing on 2022-01-01 00:00:00+00:00 9 | [2022-05-15 01:31:41,900] {standard_task_runner.py:52} INFO - Started process 2642 to run task 10 | [2022-05-15 01:31:41,921] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-01-01T00:00:00+00:00', '--job-id', '53', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpqfq21d9x', '--error-file', '/tmp/tmpffdbo0ij'] 11 | [2022-05-15 01:31:41,929] {standard_task_runner.py:77} INFO - Job 53: Subtask test 12 | [2022-05-15 01:31:42,074] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:31:42,288] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-01-01T00:00:00+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-01-01T00:00:00+00:00 19 | [2022-05-15 01:31:42,300] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:31:42,304] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:31:42,324] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:31:42,327] {subprocess.py:89} INFO - /tmp/***tmp8g34li35 24 | [2022-05-15 01:31:42,329] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:31:42,377] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220101T000000, start_date=20220515T013141, end_date=20220515T013142 26 | [2022-05-15 01:31:42,441] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:31:42,517] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 28 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-01-01T00:00:00+00:00/2.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:33:20,396] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:33:20,411] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:33:20,412] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:33:20,413] {taskinstance.py:1242} INFO - Starting attempt 2 of 2 6 | [2022-05-15 01:33:20,414] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:33:20,431] {taskinstance.py:1262} INFO - Executing on 2022-01-01 00:00:00+00:00 9 | [2022-05-15 01:33:20,437] {standard_task_runner.py:52} INFO - Started process 2747 to run task 10 | [2022-05-15 01:33:20,443] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-01-01T00:00:00+00:00', '--job-id', '62', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmp6jdkylix', '--error-file', '/tmp/tmpqzpa1c6f'] 11 | [2022-05-15 01:33:20,445] {standard_task_runner.py:77} INFO - Job 62: Subtask test 12 | [2022-05-15 01:33:20,511] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:33:20,580] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-01-01T00:00:00+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-01-01T00:00:00+00:00 19 | [2022-05-15 01:33:20,583] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:33:20,585] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd &&\n ls\n '] 22 | [2022-05-15 01:33:20,600] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:33:20,603] {subprocess.py:89} INFO - /tmp/***tmpj6tjj1en 24 | [2022-05-15 01:33:20,627] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:33:20,663] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220101T000000, start_date=20220515T013320, end_date=20220515T013320 26 | [2022-05-15 01:33:20,699] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:33:20,744] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 28 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-05-14T01:31:37.396657+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:31:41,766] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:31:41,816] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:31:41,822] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:31:41,825] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:31:41,834] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:31:41,945] {taskinstance.py:1262} INFO - Executing on 2022-05-14 01:31:37.396657+00:00 9 | [2022-05-15 01:31:41,962] {standard_task_runner.py:52} INFO - Started process 2643 to run task 10 | [2022-05-15 01:31:41,971] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-05-14T01:31:37.396657+00:00', '--job-id', '55', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpl6jr4dui', '--error-file', '/tmp/tmp5bnkiya6'] 11 | [2022-05-15 01:31:41,974] {standard_task_runner.py:77} INFO - Job 55: Subtask test 12 | [2022-05-15 01:31:42,105] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:31:42,237] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-14T01:31:37.396657+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-14T01:31:37.396657+00:00 19 | [2022-05-15 01:31:42,239] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:31:42,242] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:31:42,268] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:31:42,271] {subprocess.py:89} INFO - /tmp/***tmp68_6md08 24 | [2022-05-15 01:31:42,273] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:31:42,317] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220514T013137, start_date=20220515T013141, end_date=20220515T013142 26 | [2022-05-15 01:31:42,400] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:31:42,470] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 28 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-05-14T01:31:41.554700+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:31:43,085] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:31:43,099] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:31:43,100] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:31:43,102] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:31:43,103] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:31:43,118] {taskinstance.py:1262} INFO - Executing on 2022-05-14 01:31:41.554700+00:00 9 | [2022-05-15 01:31:43,123] {standard_task_runner.py:52} INFO - Started process 2650 to run task 10 | [2022-05-15 01:31:43,128] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-05-14T01:31:41.554700+00:00', '--job-id', '57', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmp7rhtaa1d', '--error-file', '/tmp/tmp0e736lnh'] 11 | [2022-05-15 01:31:43,129] {standard_task_runner.py:77} INFO - Job 57: Subtask test 12 | [2022-05-15 01:31:43,185] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:31:43,267] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-14T01:31:41.554700+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-14T01:31:41.554700+00:00 19 | [2022-05-15 01:31:43,271] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:31:43,272] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:31:43,292] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:31:43,294] {subprocess.py:89} INFO - /tmp/***tmp78_94enj 24 | [2022-05-15 01:31:43,295] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:31:43,337] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220514T013141, start_date=20220515T013143, end_date=20220515T013143 26 | [2022-05-15 01:31:43,384] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:31:43,441] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 28 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-05-14T01:31:44.285736+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:31:44,866] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:31:44,890] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:31:44,891] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:31:44,892] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:31:44,893] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:31:44,953] {taskinstance.py:1262} INFO - Executing on 2022-05-14 01:31:44.285736+00:00 9 | [2022-05-15 01:31:44,961] {standard_task_runner.py:52} INFO - Started process 2669 to run task 10 | [2022-05-15 01:31:44,972] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-05-14T01:31:44.285736+00:00', '--job-id', '60', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmp95lmpz5t', '--error-file', '/tmp/tmp5lqeeut9'] 11 | [2022-05-15 01:31:44,974] {standard_task_runner.py:77} INFO - Job 60: Subtask test 12 | [2022-05-15 01:31:45,095] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:31:45,336] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-14T01:31:44.285736+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-14T01:31:44.285736+00:00 19 | [2022-05-15 01:31:45,351] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:31:45,355] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:31:45,384] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:31:45,394] {subprocess.py:89} INFO - /tmp/***tmpodfsmodq 24 | [2022-05-15 01:31:45,398] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:31:45,460] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220514T013144, start_date=20220515T013144, end_date=20220515T013145 26 | [2022-05-15 01:31:45,503] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:31:45,598] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 28 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-05-14T01:33:26.251894+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:33:27,649] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:33:27,664] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:33:27,666] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:33:27,667] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:33:27,667] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:33:27,684] {taskinstance.py:1262} INFO - Executing on 2022-05-14 01:33:26.251894+00:00 9 | [2022-05-15 01:33:27,690] {standard_task_runner.py:52} INFO - Started process 2755 to run task 10 | [2022-05-15 01:33:27,696] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-05-14T01:33:26.251894+00:00', '--job-id', '64', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmp43x46jkj', '--error-file', '/tmp/tmpjioipxi_'] 11 | [2022-05-15 01:33:27,698] {standard_task_runner.py:77} INFO - Job 64: Subtask test 12 | [2022-05-15 01:33:27,755] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:33:27,815] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-14T01:33:26.251894+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-14T01:33:26.251894+00:00 19 | [2022-05-15 01:33:27,817] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:33:27,819] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd &&\n ls\n '] 22 | [2022-05-15 01:33:27,833] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:33:27,835] {subprocess.py:89} INFO - /tmp/***tmph5bh6vmq 24 | [2022-05-15 01:33:27,844] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:33:27,878] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220514T013326, start_date=20220515T013327, end_date=20220515T013327 26 | [2022-05-15 01:33:27,911] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:33:27,980] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 28 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2021-12-24T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:14,056] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:30:15,995] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:30:16,007] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:30:16,018] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:30:16,021] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:30:17,658] {taskinstance.py:1262} INFO - Executing on 2021-12-24 00:00:00+00:00 9 | [2022-05-15 01:30:18,123] {standard_task_runner.py:52} INFO - Started process 2574 to run task 10 | [2022-05-15 01:30:18,209] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2021-12-24T00:00:00+00:00', '--job-id', '38', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmp2wmq4yrk', '--error-file', '/tmp/tmpry141fdk'] 11 | [2022-05-15 01:30:18,262] {standard_task_runner.py:77} INFO - Job 38: Subtask test 12 | [2022-05-15 01:30:24,600] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:30:28,986] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2021-12-24T00:00:00+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2021-12-24T00:00:00+00:00 19 | [2022-05-15 01:30:29,042] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:30:30,713] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:30:33,310] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:30:34,293] {subprocess.py:89} INFO - /tmp/***tmpj6erl6lm 24 | [2022-05-15 01:30:34,296] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:30:35,658] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20211224T000000, start_date=20220515T013014, end_date=20220515T013035 26 | [2022-05-15 01:30:37,251] {local_task_job.py:212} WARNING - State of this instance has been externally set to success. Terminating instance. 27 | [2022-05-15 01:30:37,423] {process_utils.py:100} INFO - Sending Signals.SIGTERM to GPID 2574 28 | [2022-05-15 01:30:37,467] {process_utils.py:66} INFO - Process psutil.Process(pid=2574, status='terminated', exitcode=0, started='01:30:17') (2574) terminated with exit code 0 29 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.assert_positive_total_for_payments/2022-05-17T22:12:00.138285+00:00/5.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:40:29,103] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:40:29,131] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:40:29,133] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:40:29,135] {taskinstance.py:1242} INFO - Starting attempt 5 of 6 6 | [2022-05-18 23:40:29,136] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:40:29,167] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:40:29,186] {standard_task_runner.py:52} INFO - Started process 915 to run task 10 | [2022-05-18 23:40:29,208] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.assert_positive_total_for_payments', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '183', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmpidlzkzu4', '--error-file', '/tmp/tmpkfc61no0'] 11 | [2022-05-18 23:40:29,210] {standard_task_runner.py:77} INFO - Job 183: Subtask dbt_test.test.jaffle_shop.assert_positive_total_for_payments 12 | [2022-05-18 23:40:29,426] {logging_mixin.py:109} INFO - Running on host d4b673e28191 13 | [2022-05-18 23:40:30,106] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.assert_positive_total_for_payments 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:40:30,109] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:40:30,116] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select assert_positive_total_for_payments --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:40:30,263] {subprocess.py:85} INFO - Output: 23 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c/2022-05-17T22:12:00.138285+00:00/6.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:40:35,399] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:40:35,427] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:40:35,429] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:40:35,431] {taskinstance.py:1242} INFO - Starting attempt 6 of 7 6 | [2022-05-18 23:40:35,433] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:40:35,464] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:40:35,482] {standard_task_runner.py:52} INFO - Started process 930 to run task 10 | [2022-05-18 23:40:35,506] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '189', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmp2r39fbpn', '--error-file', '/tmp/tmpffpr0vs1'] 11 | [2022-05-18 23:40:35,509] {standard_task_runner.py:77} INFO - Job 189: Subtask dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c 12 | [2022-05-18 23:40:35,755] {logging_mixin.py:109} INFO - Running on host d4b673e28191 13 | [2022-05-18 23:40:35,926] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:40:35,929] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:40:35,933] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select source_not_null_stripe_payment_id --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:40:36,022] {subprocess.py:85} INFO - Output: 23 | -------------------------------------------------------------------------------- /superset/docker/docker-init.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | set -e 19 | 20 | # 21 | # Always install local overrides first 22 | # 23 | /app/docker/docker-bootstrap.sh 24 | 25 | STEP_CNT=4 26 | 27 | echo_step() { 28 | cat < 2 | [2022-05-18 23:40:29,322] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:40:29,331] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:40:29,338] {taskinstance.py:1242} INFO - Starting attempt 5 of 6 6 | [2022-05-18 23:40:29,349] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:40:29,418] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:40:29,610] {standard_task_runner.py:52} INFO - Started process 916 to run task 10 | [2022-05-18 23:40:29,658] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.source_unique_jaffle_shop_orders_id.8a425b2fba', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '184', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmpno1u_u7d', '--error-file', '/tmp/tmpf_2e36q5'] 11 | [2022-05-18 23:40:29,667] {standard_task_runner.py:77} INFO - Job 184: Subtask dbt_test.test.jaffle_shop.source_unique_jaffle_shop_orders_id.8a425b2fba 12 | [2022-05-18 23:40:30,191] {logging_mixin.py:109} INFO - Running on host d4b673e28191 13 | [2022-05-18 23:40:30,602] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.source_unique_jaffle_shop_orders_id.8a425b2fba 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:40:30,607] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:40:30,613] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select source_unique_jaffle_shop_orders_id --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:40:30,749] {subprocess.py:85} INFO - Output: 23 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-05-14T01:31:44.285736+00:00/2.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:39:31,135] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:39:31,155] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:39:31,156] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:39:31,157] {taskinstance.py:1242} INFO - Starting attempt 2 of 2 6 | [2022-05-15 01:39:31,158] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:39:31,176] {taskinstance.py:1262} INFO - Executing on 2022-05-14 01:31:44.285736+00:00 9 | [2022-05-15 01:39:31,184] {standard_task_runner.py:52} INFO - Started process 3060 to run task 10 | [2022-05-15 01:39:31,190] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-05-14T01:31:44.285736+00:00', '--job-id', '69', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpglhz6_k9', '--error-file', '/tmp/tmpe_409_ts'] 11 | [2022-05-15 01:39:31,191] {standard_task_runner.py:77} INFO - Job 69: Subtask test 12 | [2022-05-15 01:39:31,274] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:39:31,382] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-14T01:31:44.285736+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-14T01:31:44.285736+00:00 19 | [2022-05-15 01:39:31,388] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:39:31,392] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n ls /opt/***/\n '] 22 | [2022-05-15 01:39:31,432] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:39:31,453] {subprocess.py:89} INFO - ***-worker.pid 24 | [2022-05-15 01:39:31,456] {subprocess.py:89} INFO - ***.cfg 25 | [2022-05-15 01:39:31,458] {subprocess.py:89} INFO - dags 26 | [2022-05-15 01:39:31,461] {subprocess.py:89} INFO - logs 27 | [2022-05-15 01:39:31,463] {subprocess.py:89} INFO - plugins 28 | [2022-05-15 01:39:31,464] {subprocess.py:89} INFO - webserver_config.py 29 | [2022-05-15 01:39:31,468] {subprocess.py:93} INFO - Command exited with return code 0 30 | [2022-05-15 01:39:31,553] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220514T013144, start_date=20220515T013931, end_date=20220515T013931 31 | [2022-05-15 01:39:31,610] {local_task_job.py:154} INFO - Task exited with return code 0 32 | [2022-05-15 01:39:31,718] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 33 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-05-14T01:33:26.251894+00:00/4.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:40:03,382] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:40:03,400] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:40:03,402] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:40:03,402] {taskinstance.py:1242} INFO - Starting attempt 4 of 4 6 | [2022-05-15 01:40:03,403] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:40:03,420] {taskinstance.py:1262} INFO - Executing on 2022-05-14 01:33:26.251894+00:00 9 | [2022-05-15 01:40:03,426] {standard_task_runner.py:52} INFO - Started process 3098 to run task 10 | [2022-05-15 01:40:03,434] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-05-14T01:33:26.251894+00:00', '--job-id', '71', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmp8o26jg_1', '--error-file', '/tmp/tmp6hn8hz9x'] 11 | [2022-05-15 01:40:03,436] {standard_task_runner.py:77} INFO - Job 71: Subtask test 12 | [2022-05-15 01:40:03,505] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:40:03,588] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-14T01:33:26.251894+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-14T01:33:26.251894+00:00 19 | [2022-05-15 01:40:03,593] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:40:03,596] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n ls /opt/***/\n '] 22 | [2022-05-15 01:40:03,616] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:40:03,621] {subprocess.py:89} INFO - ***-worker.pid 24 | [2022-05-15 01:40:03,622] {subprocess.py:89} INFO - ***.cfg 25 | [2022-05-15 01:40:03,623] {subprocess.py:89} INFO - dags 26 | [2022-05-15 01:40:03,625] {subprocess.py:89} INFO - logs 27 | [2022-05-15 01:40:03,626] {subprocess.py:89} INFO - plugins 28 | [2022-05-15 01:40:03,627] {subprocess.py:89} INFO - webserver_config.py 29 | [2022-05-15 01:40:03,628] {subprocess.py:93} INFO - Command exited with return code 0 30 | [2022-05-15 01:40:03,657] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220514T013326, start_date=20220515T014003, end_date=20220515T014003 31 | [2022-05-15 01:40:03,690] {local_task_job.py:154} INFO - Task exited with return code 0 32 | [2022-05-15 01:40:03,736] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 33 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-05-14T01:33:26.251894+00:00/5.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:41:02,369] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:41:02,389] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:41:02,391] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:41:02,392] {taskinstance.py:1242} INFO - Starting attempt 5 of 5 6 | [2022-05-15 01:41:02,393] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:41:02,408] {taskinstance.py:1262} INFO - Executing on 2022-05-14 01:33:26.251894+00:00 9 | [2022-05-15 01:41:02,413] {standard_task_runner.py:52} INFO - Started process 3148 to run task 10 | [2022-05-15 01:41:02,418] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-05-14T01:33:26.251894+00:00', '--job-id', '73', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmp11x09aie', '--error-file', '/tmp/tmpgjk0z34f'] 11 | [2022-05-15 01:41:02,419] {standard_task_runner.py:77} INFO - Job 73: Subtask test 12 | [2022-05-15 01:41:02,482] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:41:02,545] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-14T01:33:26.251894+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-14T01:33:26.251894+00:00 19 | [2022-05-15 01:41:02,547] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:41:02,549] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n ls /opt/***/\n '] 22 | [2022-05-15 01:41:02,562] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:41:02,566] {subprocess.py:89} INFO - ***-worker.pid 24 | [2022-05-15 01:41:02,568] {subprocess.py:89} INFO - ***.cfg 25 | [2022-05-15 01:41:02,568] {subprocess.py:89} INFO - dags 26 | [2022-05-15 01:41:02,569] {subprocess.py:89} INFO - logs 27 | [2022-05-15 01:41:02,570] {subprocess.py:89} INFO - plugins 28 | [2022-05-15 01:41:02,571] {subprocess.py:89} INFO - webserver_config.py 29 | [2022-05-15 01:41:02,572] {subprocess.py:93} INFO - Command exited with return code 0 30 | [2022-05-15 01:41:02,601] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220514T013326, start_date=20220515T014102, end_date=20220515T014102 31 | [2022-05-15 01:41:02,638] {local_task_job.py:154} INFO - Task exited with return code 0 32 | [2022-05-15 01:41:02,689] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 33 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-05-14T01:33:26.251894+00:00/6.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:41:50,660] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:41:50,675] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:41:50,676] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:41:50,677] {taskinstance.py:1242} INFO - Starting attempt 6 of 6 6 | [2022-05-15 01:41:50,679] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:41:50,707] {taskinstance.py:1262} INFO - Executing on 2022-05-14 01:33:26.251894+00:00 9 | [2022-05-15 01:41:50,717] {standard_task_runner.py:52} INFO - Started process 3197 to run task 10 | [2022-05-15 01:41:50,721] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-05-14T01:33:26.251894+00:00', '--job-id', '75', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpyqk3leow', '--error-file', '/tmp/tmpjgfflnm_'] 11 | [2022-05-15 01:41:50,722] {standard_task_runner.py:77} INFO - Job 75: Subtask test 12 | [2022-05-15 01:41:50,773] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:41:50,829] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-14T01:33:26.251894+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-14T01:33:26.251894+00:00 19 | [2022-05-15 01:41:50,831] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:41:50,833] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n ls /opt/***/\n '] 22 | [2022-05-15 01:41:50,845] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:41:50,853] {subprocess.py:89} INFO - ***-worker.pid 24 | [2022-05-15 01:41:50,854] {subprocess.py:89} INFO - ***.cfg 25 | [2022-05-15 01:41:50,854] {subprocess.py:89} INFO - dags 26 | [2022-05-15 01:41:50,855] {subprocess.py:89} INFO - logs 27 | [2022-05-15 01:41:50,856] {subprocess.py:89} INFO - plugins 28 | [2022-05-15 01:41:50,856] {subprocess.py:89} INFO - webserver_config.py 29 | [2022-05-15 01:41:50,858] {subprocess.py:93} INFO - Command exited with return code 0 30 | [2022-05-15 01:41:50,887] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220514T013326, start_date=20220515T014150, end_date=20220515T014150 31 | [2022-05-15 01:41:50,938] {local_task_job.py:154} INFO - Task exited with return code 0 32 | [2022-05-15 01:41:51,046] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 33 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-05-14T01:33:26.251894+00:00/7.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:42:50,779] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:42:50,812] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:42:50,814] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:42:50,815] {taskinstance.py:1242} INFO - Starting attempt 7 of 7 6 | [2022-05-15 01:42:50,817] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:42:50,836] {taskinstance.py:1262} INFO - Executing on 2022-05-14 01:33:26.251894+00:00 9 | [2022-05-15 01:42:50,843] {standard_task_runner.py:52} INFO - Started process 3251 to run task 10 | [2022-05-15 01:42:50,847] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-05-14T01:33:26.251894+00:00', '--job-id', '77', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpgfxe4_6d', '--error-file', '/tmp/tmpek40ep1w'] 11 | [2022-05-15 01:42:50,849] {standard_task_runner.py:77} INFO - Job 77: Subtask test 12 | [2022-05-15 01:42:50,905] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:42:50,981] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-14T01:33:26.251894+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-14T01:33:26.251894+00:00 19 | [2022-05-15 01:42:50,984] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:42:50,986] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n ls /opt/***/\n '] 22 | [2022-05-15 01:42:51,001] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:42:51,006] {subprocess.py:89} INFO - ***-worker.pid 24 | [2022-05-15 01:42:51,007] {subprocess.py:89} INFO - ***.cfg 25 | [2022-05-15 01:42:51,008] {subprocess.py:89} INFO - dags 26 | [2022-05-15 01:42:51,009] {subprocess.py:89} INFO - logs 27 | [2022-05-15 01:42:51,010] {subprocess.py:89} INFO - plugins 28 | [2022-05-15 01:42:51,011] {subprocess.py:89} INFO - webserver_config.py 29 | [2022-05-15 01:42:51,012] {subprocess.py:93} INFO - Command exited with return code 0 30 | [2022-05-15 01:42:51,041] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220514T013326, start_date=20220515T014250, end_date=20220515T014251 31 | [2022-05-15 01:42:51,103] {local_task_job.py:154} INFO - Task exited with return code 0 32 | [2022-05-15 01:42:51,156] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 33 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.source_unique_jaffle_shop_customers_id.2777a7933e/2022-05-17T22:12:00.138285+00:00/5.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:40:30,130] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:40:30,211] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:40:30,214] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:40:30,223] {taskinstance.py:1242} INFO - Starting attempt 5 of 6 6 | [2022-05-18 23:40:30,232] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:40:30,294] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:40:30,326] {standard_task_runner.py:52} INFO - Started process 918 to run task 10 | [2022-05-18 23:40:30,331] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.source_unique_jaffle_shop_customers_id.2777a7933e', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '185', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmpr2texnti', '--error-file', '/tmp/tmp87566nhn'] 11 | [2022-05-18 23:40:30,345] {standard_task_runner.py:77} INFO - Job 185: Subtask dbt_test.test.jaffle_shop.source_unique_jaffle_shop_customers_id.2777a7933e 12 | [2022-05-18 23:40:30,656] {logging_mixin.py:109} INFO - Running on host d4b673e28191 13 | [2022-05-18 23:40:30,874] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.source_unique_jaffle_shop_customers_id.2777a7933e 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:40:30,880] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:40:30,885] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select source_unique_jaffle_shop_customers_id --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:40:30,953] {subprocess.py:85} INFO - Output: 23 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.not_null_stg_jaffle_shop__orders_order_id.aad0410d08/2022-05-17T22:12:00.138285+00:00/2.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:02:39,051] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:02:39,186] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:02:39,187] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:02:39,190] {taskinstance.py:1242} INFO - Starting attempt 2 of 5 6 | [2022-05-18 23:02:39,192] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:02:39,440] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:02:39,547] {standard_task_runner.py:52} INFO - Started process 412 to run task 10 | [2022-05-18 23:02:39,577] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.not_null_stg_jaffle_shop__orders_order_id.aad0410d08', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '141', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmpiq67mo6n', '--error-file', '/tmp/tmps1oc2d76'] 11 | [2022-05-18 23:02:39,579] {standard_task_runner.py:77} INFO - Job 141: Subtask dbt_test.test.jaffle_shop.not_null_stg_jaffle_shop__orders_order_id.aad0410d08 12 | [2022-05-18 23:02:40,133] {logging_mixin.py:109} INFO - Running on host 3d7cb1bfa13d 13 | [2022-05-18 23:02:40,462] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.not_null_stg_jaffle_shop__orders_order_id.aad0410d08 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:02:40,476] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:02:40,485] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select not_null_stg_jaffle_shop__orders_order_id --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:02:40,771] {subprocess.py:85} INFO - Output: 23 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.not_null_stg_jaffle_shop__orders_order_id.aad0410d08/2022-05-17T22:12:00.138285+00:00/4.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:35:28,240] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:35:29,178] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:35:29,351] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:35:29,383] {taskinstance.py:1242} INFO - Starting attempt 4 of 6 6 | [2022-05-18 23:35:29,390] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:35:36,612] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:35:36,898] {standard_task_runner.py:52} INFO - Started process 579 to run task 10 | [2022-05-18 23:35:36,929] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.not_null_stg_jaffle_shop__orders_order_id.aad0410d08', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '174', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmphqoi07tw', '--error-file', '/tmp/tmpay542dd1'] 11 | [2022-05-18 23:35:36,948] {standard_task_runner.py:77} INFO - Job 174: Subtask dbt_test.test.jaffle_shop.not_null_stg_jaffle_shop__orders_order_id.aad0410d08 12 | [2022-05-18 23:35:44,702] {logging_mixin.py:109} INFO - Running on host d4b673e28191 13 | [2022-05-18 23:36:01,175] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.not_null_stg_jaffle_shop__orders_order_id.aad0410d08 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:36:01,205] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:36:01,217] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select not_null_stg_jaffle_shop__orders_order_id --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:36:01,602] {subprocess.py:85} INFO - Output: 23 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.source_unique_stripe_payment_id.3b7989ce64/2022-05-17T22:12:00.138285+00:00/5.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:35:20,950] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:35:20,990] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:35:20,993] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:35:20,996] {taskinstance.py:1242} INFO - Starting attempt 5 of 7 6 | [2022-05-18 23:35:20,999] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:35:21,084] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:35:21,161] {standard_task_runner.py:52} INFO - Started process 568 to run task 10 | [2022-05-18 23:35:21,207] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.source_unique_stripe_payment_id.3b7989ce64', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '167', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmpacnz6bl2', '--error-file', '/tmp/tmplol6lek2'] 11 | [2022-05-18 23:35:21,220] {standard_task_runner.py:77} INFO - Job 167: Subtask dbt_test.test.jaffle_shop.source_unique_stripe_payment_id.3b7989ce64 12 | [2022-05-18 23:35:22,335] {logging_mixin.py:109} INFO - Running on host d4b673e28191 13 | [2022-05-18 23:35:23,958] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.source_unique_stripe_payment_id.3b7989ce64 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:35:24,128] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:35:24,134] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select source_unique_stripe_payment_id --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:35:24,388] {subprocess.py:85} INFO - Output: 23 | [2022-05-18 23:37:38,328] {local_task_job.py:154} INFO - Task exited with return code Negsignal.SIGKILL 24 | -------------------------------------------------------------------------------- /db/data/jaffle_shop_orders.csv: -------------------------------------------------------------------------------- 1 | ID,USER_ID,ORDER_DATE,STATUS 2 | 1,1,2018-01-01,returned 3 | 2,3,2018-01-02,completed 4 | 3,94,2018-01-04,completed 5 | 4,50,2018-01-05,completed 6 | 5,64,2018-01-05,completed 7 | 6,54,2018-01-07,completed 8 | 7,88,2018-01-09,completed 9 | 8,2,2018-01-11,returned 10 | 9,53,2018-01-12,completed 11 | 10,7,2018-01-14,completed 12 | 11,99,2018-01-14,completed 13 | 12,59,2018-01-15,completed 14 | 13,84,2018-01-17,completed 15 | 14,40,2018-01-17,returned 16 | 15,25,2018-01-17,completed 17 | 16,39,2018-01-18,completed 18 | 17,71,2018-01-18,completed 19 | 18,64,2018-01-20,returned 20 | 19,54,2018-01-22,completed 21 | 20,20,2018-01-23,completed 22 | 21,71,2018-01-23,completed 23 | 22,86,2018-01-24,completed 24 | 23,22,2018-01-26,return_pending 25 | 24,3,2018-01-27,completed 26 | 25,51,2018-01-28,completed 27 | 26,32,2018-01-28,completed 28 | 27,94,2018-01-29,completed 29 | 28,8,2018-01-29,completed 30 | 29,57,2018-01-31,completed 31 | 30,69,2018-02-02,completed 32 | 31,16,2018-02-02,completed 33 | 32,28,2018-02-04,completed 34 | 33,42,2018-02-04,completed 35 | 34,38,2018-02-06,completed 36 | 35,80,2018-02-08,completed 37 | 36,85,2018-02-10,completed 38 | 37,1,2018-02-10,completed 39 | 38,51,2018-02-10,completed 40 | 39,26,2018-02-11,completed 41 | 40,33,2018-02-13,completed 42 | 41,99,2018-02-14,completed 43 | 42,92,2018-02-16,completed 44 | 43,31,2018-02-17,completed 45 | 44,66,2018-02-17,completed 46 | 45,22,2018-02-17,completed 47 | 46,6,2018-02-19,completed 48 | 47,50,2018-02-20,completed 49 | 48,27,2018-02-21,completed 50 | 49,35,2018-02-21,completed 51 | 50,51,2018-02-23,completed 52 | 51,71,2018-02-24,completed 53 | 52,54,2018-02-25,return_pending 54 | 53,34,2018-02-26,completed 55 | 54,54,2018-02-26,completed 56 | 55,18,2018-02-27,completed 57 | 56,79,2018-02-28,completed 58 | 57,93,2018-03-01,completed 59 | 58,22,2018-03-01,completed 60 | 59,30,2018-03-02,completed 61 | 60,12,2018-03-03,completed 62 | 61,63,2018-03-03,completed 63 | 62,57,2018-03-05,completed 64 | 63,70,2018-03-06,completed 65 | 64,13,2018-03-07,completed 66 | 65,26,2018-03-08,completed 67 | 66,36,2018-03-10,completed 68 | 67,79,2018-03-11,completed 69 | 68,53,2018-03-11,completed 70 | 69,3,2018-03-11,completed 71 | 70,8,2018-03-12,completed 72 | 71,42,2018-03-12,shipped 73 | 72,30,2018-03-14,shipped 74 | 73,19,2018-03-16,completed 75 | 74,9,2018-03-17,shipped 76 | 75,69,2018-03-18,completed 77 | 76,25,2018-03-20,completed 78 | 77,35,2018-03-21,shipped 79 | 78,90,2018-03-23,shipped 80 | 79,52,2018-03-23,shipped 81 | 80,11,2018-03-23,shipped 82 | 81,76,2018-03-23,shipped 83 | 82,46,2018-03-24,shipped 84 | 83,54,2018-03-24,shipped 85 | 84,70,2018-03-26,placed 86 | 85,47,2018-03-26,shipped 87 | 86,68,2018-03-26,placed 88 | 87,46,2018-03-27,placed 89 | 88,91,2018-03-27,shipped 90 | 89,21,2018-03-28,placed 91 | 90,66,2018-03-30,shipped 92 | 91,47,2018-03-31,placed 93 | 92,84,2018-04-02,placed 94 | 93,66,2018-04-03,placed 95 | 94,63,2018-04-03,placed 96 | 95,27,2018-04-04,placed 97 | 96,90,2018-04-06,placed 98 | 97,89,2018-04-07,placed 99 | 98,41,2018-04-07,placed 100 | 99,85,2018-04-09,placed -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2022-05-14T01:33:26.251894+00:00/3.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:37:40,424] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:37:40,438] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:37:40,439] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:37:40,440] {taskinstance.py:1242} INFO - Starting attempt 3 of 3 6 | [2022-05-15 01:37:40,440] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:37:40,456] {taskinstance.py:1262} INFO - Executing on 2022-05-14 01:33:26.251894+00:00 9 | [2022-05-15 01:37:40,462] {standard_task_runner.py:52} INFO - Started process 2973 to run task 10 | [2022-05-15 01:37:40,468] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2022-05-14T01:33:26.251894+00:00', '--job-id', '67', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpgr6fhw4p', '--error-file', '/tmp/tmpiyygfjdc'] 11 | [2022-05-15 01:37:40,469] {standard_task_runner.py:77} INFO - Job 67: Subtask test 12 | [2022-05-15 01:37:40,527] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:37:40,589] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-14T01:33:26.251894+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-14T01:33:26.251894+00:00 19 | [2022-05-15 01:37:40,591] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:37:40,593] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n ls /usr/local/\n '] 22 | [2022-05-15 01:37:40,606] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:37:40,612] {subprocess.py:89} INFO - bin 24 | [2022-05-15 01:37:40,613] {subprocess.py:89} INFO - etc 25 | [2022-05-15 01:37:40,614] {subprocess.py:89} INFO - games 26 | [2022-05-15 01:37:40,615] {subprocess.py:89} INFO - include 27 | [2022-05-15 01:37:40,616] {subprocess.py:89} INFO - lib 28 | [2022-05-15 01:37:40,617] {subprocess.py:89} INFO - man 29 | [2022-05-15 01:37:40,618] {subprocess.py:89} INFO - sbin 30 | [2022-05-15 01:37:40,619] {subprocess.py:89} INFO - share 31 | [2022-05-15 01:37:40,620] {subprocess.py:89} INFO - src 32 | [2022-05-15 01:37:40,621] {subprocess.py:93} INFO - Command exited with return code 0 33 | [2022-05-15 01:37:40,652] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20220514T013326, start_date=20220515T013740, end_date=20220515T013740 34 | [2022-05-15 01:37:40,683] {local_task_job.py:154} INFO - Task exited with return code 0 35 | [2022-05-15 01:37:40,734] {local_task_job.py:264} INFO - 1 downstream tasks scheduled from follow-on schedule check 36 | -------------------------------------------------------------------------------- /superset/docker-compose-non-dev.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | x-superset-image: &superset-image apache/superset:${TAG:-latest-dev} 18 | x-superset-depends-on: &superset-depends-on 19 | - db 20 | - redis 21 | x-superset-volumes: &superset-volumes 22 | # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container 23 | - ./docker:/app/docker 24 | - superset_home:/app/superset_home 25 | 26 | version: "3.7" 27 | services: 28 | redis: 29 | image: redis:latest 30 | container_name: superset_cache 31 | restart: unless-stopped 32 | volumes: 33 | - redis:/data 34 | 35 | db: 36 | env_file: docker/.env-non-dev 37 | image: postgres:10 38 | container_name: superset_db 39 | restart: unless-stopped 40 | volumes: 41 | - db_home:/var/lib/postgresql/data 42 | 43 | superset: 44 | env_file: docker/.env-non-dev 45 | image: *superset-image 46 | container_name: superset_app 47 | command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"] 48 | user: "root" 49 | restart: unless-stopped 50 | ports: 51 | - 8088:8088 52 | depends_on: *superset-depends-on 53 | volumes: *superset-volumes 54 | 55 | superset-init: 56 | image: *superset-image 57 | container_name: superset_init 58 | command: ["/app/docker/docker-init.sh"] 59 | env_file: docker/.env-non-dev 60 | depends_on: *superset-depends-on 61 | user: "root" 62 | volumes: *superset-volumes 63 | 64 | superset-worker: 65 | image: *superset-image 66 | container_name: superset_worker 67 | command: ["/app/docker/docker-bootstrap.sh", "worker"] 68 | env_file: docker/.env-non-dev 69 | restart: unless-stopped 70 | depends_on: *superset-depends-on 71 | user: "root" 72 | volumes: *superset-volumes 73 | 74 | superset-worker-beat: 75 | image: *superset-image 76 | container_name: superset_worker_beat 77 | command: ["/app/docker/docker-bootstrap.sh", "beat"] 78 | env_file: docker/.env-non-dev 79 | restart: unless-stopped 80 | depends_on: *superset-depends-on 81 | user: "root" 82 | volumes: *superset-volumes 83 | 84 | volumes: 85 | superset_home: 86 | external: false 87 | db_home: 88 | external: false 89 | redis: 90 | external: false 91 | 92 | networks: 93 | default: 94 | external: true 95 | name: open-source-data-stack_default -------------------------------------------------------------------------------- /airflow/dags/model_level_dbt_dag.py: -------------------------------------------------------------------------------- 1 | from airflow import DAG 2 | from airflow.operators.dummy import DummyOperator 3 | from airflow.operators.bash import BashOperator 4 | from airflow.utils.task_group import TaskGroup 5 | from airflow.utils.dates import datetime, timedelta 6 | from dbt_dag_parser_v2 import DbtDagParser 7 | 8 | DAG_ID = "model_level_dbt_dag" 9 | DAG_OWNER = "luciano.naveiro" 10 | 11 | DBT_PROJECT_PATH = "/opt/dbt/jaffle_shop" 12 | DBT_GLOBAL_CLI_FLAGS = "--no-write-json" 13 | DBT_TARGET = "dev" 14 | 15 | default_args = { 16 | "owner": DAG_OWNER, 17 | "retries": 3, 18 | "retry_delay": timedelta(minutes=2), 19 | } 20 | 21 | 22 | with DAG( 23 | DAG_ID, 24 | start_date=datetime(2021, 12, 23), 25 | description="A dbt wrapper for Airflow using a utility class to map the dbt DAG to Airflow tasks", 26 | schedule_interval=timedelta(days=1), 27 | default_args=default_args, 28 | catchup=False, 29 | ) as dag: 30 | 31 | # Define DummyOperator 32 | start_dummy = DummyOperator(task_id="start") 33 | end_dummy = DummyOperator(task_id="end") 34 | start_run_dbt_dummy = DummyOperator(task_id="start_run_dbt") 35 | start_test_dbt_dummy = DummyOperator(task_id="start_test_dbt") 36 | 37 | # Validate taskgroups 38 | validate_taskgroup = TaskGroup("dbt_validate") 39 | dbt_debug = BashOperator( 40 | task_id="dbt_debug", 41 | task_group=validate_taskgroup, 42 | bash_command=f""" 43 | dbt debug --profiles-dir {DBT_PROJECT_PATH} --project-dir {DBT_PROJECT_PATH} 44 | """, 45 | ) 46 | 47 | dbt_parse = BashOperator( 48 | task_id="dbt_parse", 49 | task_group=validate_taskgroup, 50 | bash_command=f""" 51 | dbt parse --profiles-dir {DBT_PROJECT_PATH} --project-dir {DBT_PROJECT_PATH} 52 | """, 53 | ) 54 | 55 | dbt_compile = BashOperator( 56 | task_id="dbt_compile", 57 | task_group=validate_taskgroup, 58 | bash_command=f""" 59 | dbt compile --profiles-dir {DBT_PROJECT_PATH} --project-dir {DBT_PROJECT_PATH} --vars '{{"date": " {{{{ ds }}}} " }}' 60 | """, 61 | ) 62 | 63 | start_dummy >> dbt_debug >> dbt_parse >> dbt_compile >> start_run_dbt_dummy 64 | 65 | # The parser parses out a dbt manifest.json file and dynamically creates tasks for "dbt run" and "dbt test" 66 | # commands for each individual model. It groups them into task groups which we can retrieve and use in the DAG. 67 | dag_parser = DbtDagParser( 68 | dbt_global_cli_flags=DBT_GLOBAL_CLI_FLAGS, 69 | dbt_project_dir=DBT_PROJECT_PATH, 70 | dbt_profiles_dir=DBT_PROJECT_PATH, 71 | dbt_target=DBT_TARGET, 72 | ) 73 | dbt_run_group = dag_parser.get_dbt_run_group() 74 | dbt_test_group = dag_parser.get_dbt_test_group() 75 | 76 | start_run_dbt_dummy >> dbt_run_group >> start_test_dbt_dummy >> dbt_test_group 77 | 78 | dbt_docs = BashOperator( 79 | task_id="dbt_docs", 80 | bash_command=f""" 81 | dbt docs generate --profiles-dir {DBT_PROJECT_PATH} --project-dir {DBT_PROJECT_PATH} --vars '{{"date": " {{{{ ds }}}} " }}' 82 | """, 83 | ) 84 | 85 | dbt_test_group >> dbt_docs >> end_dummy 86 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2021-12-23T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:14,329] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:30:15,915] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:30:15,920] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:30:15,925] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:30:15,927] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:30:16,157] {taskinstance.py:1262} INFO - Executing on 2021-12-23 00:00:00+00:00 9 | [2022-05-15 01:30:17,774] {standard_task_runner.py:52} INFO - Started process 2573 to run task 10 | [2022-05-15 01:30:17,972] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2021-12-23T00:00:00+00:00', '--job-id', '39', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpt0uqm9gq', '--error-file', '/tmp/tmp8mkkm4w6'] 11 | [2022-05-15 01:30:17,983] {standard_task_runner.py:77} INFO - Job 39: Subtask test 12 | [2022-05-15 01:30:25,627] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:30:31,806] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2021-12-23T00:00:00+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2021-12-23T00:00:00+00:00 19 | [2022-05-15 01:30:31,901] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:30:32,235] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:30:34,324] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:30:34,336] {subprocess.py:89} INFO - /tmp/***tmp8y7rldx2 24 | [2022-05-15 01:30:34,353] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:30:35,651] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20211223T000000, start_date=20220515T013014, end_date=20220515T013035 26 | [2022-05-15 01:30:37,439] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:30:38,452] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 28 | [2022-05-15 01:30:38,632] {dagrun.py:527} ERROR - Marking run failed 29 | [2022-05-15 01:30:38,650] {dagrun.py:602} INFO - DagRun Finished: dag_id=dbt_dag, execution_date=2021-12-23 00:00:00+00:00, run_id=scheduled__2021-12-23T00:00:00+00:00, run_start_date=2022-05-15 01:30:07.127612+00:00, run_end_date=2022-05-15 01:30:38.649805+00:00, run_duration=31.522193, state=failed, external_trigger=False, run_type=scheduled, data_interval_start=2021-12-23 00:00:00+00:00, data_interval_end=2021-12-24 00:00:00+00:00, dag_hash=08ba76bc4de643e0df82526ee6b4f9f1 30 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2021-12-25T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:19,052] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:30:20,518] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:30:20,551] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:30:20,558] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:30:20,587] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:30:21,957] {taskinstance.py:1262} INFO - Executing on 2021-12-25 00:00:00+00:00 9 | [2022-05-15 01:30:24,578] {standard_task_runner.py:52} INFO - Started process 2578 to run task 10 | [2022-05-15 01:30:24,658] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2021-12-25T00:00:00+00:00', '--job-id', '43', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmp36nzs23h', '--error-file', '/tmp/tmpjp5p9dzs'] 11 | [2022-05-15 01:30:24,664] {standard_task_runner.py:77} INFO - Job 43: Subtask test 12 | [2022-05-15 01:30:25,855] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:30:32,562] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2021-12-25T00:00:00+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2021-12-25T00:00:00+00:00 19 | [2022-05-15 01:30:32,749] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:30:32,774] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:30:34,030] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:30:34,126] {subprocess.py:89} INFO - /tmp/***tmpv9mtziok 24 | [2022-05-15 01:30:34,183] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:30:35,564] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20211225T000000, start_date=20220515T013019, end_date=20220515T013035 26 | [2022-05-15 01:30:36,538] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:30:38,091] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 28 | [2022-05-15 01:30:38,473] {dagrun.py:527} ERROR - Marking run failed 29 | [2022-05-15 01:30:38,496] {dagrun.py:602} INFO - DagRun Finished: dag_id=dbt_dag, execution_date=2021-12-25 00:00:00+00:00, run_id=scheduled__2021-12-25T00:00:00+00:00, run_start_date=2022-05-15 01:30:08.361776+00:00, run_end_date=2022-05-15 01:30:38.495690+00:00, run_duration=30.133914, state=failed, external_trigger=False, run_type=scheduled, data_interval_start=2021-12-25 00:00:00+00:00, data_interval_end=2021-12-26 00:00:00+00:00, dag_hash=08ba76bc4de643e0df82526ee6b4f9f1 30 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2021-12-26T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:34,785] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:30:35,601] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:30:35,609] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:30:35,613] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:30:35,624] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:30:36,152] {taskinstance.py:1262} INFO - Executing on 2021-12-26 00:00:00+00:00 9 | [2022-05-15 01:30:36,462] {standard_task_runner.py:52} INFO - Started process 2591 to run task 10 | [2022-05-15 01:30:36,753] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2021-12-26T00:00:00+00:00', '--job-id', '44', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmprigi7phs', '--error-file', '/tmp/tmpi5e32bju'] 11 | [2022-05-15 01:30:36,814] {standard_task_runner.py:77} INFO - Job 44: Subtask test 12 | [2022-05-15 01:30:37,420] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:30:38,760] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2021-12-26T00:00:00+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2021-12-26T00:00:00+00:00 19 | [2022-05-15 01:30:38,767] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:30:38,774] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:30:38,876] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:30:38,888] {subprocess.py:89} INFO - /tmp/***tmp9clp157v 24 | [2022-05-15 01:30:38,901] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:30:39,285] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20211226T000000, start_date=20220515T013035, end_date=20220515T013039 26 | [2022-05-15 01:30:48,024] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:30:48,744] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 28 | [2022-05-15 01:30:48,868] {dagrun.py:527} ERROR - Marking run failed 29 | [2022-05-15 01:30:48,884] {dagrun.py:602} INFO - DagRun Finished: dag_id=dbt_dag, execution_date=2021-12-26 00:00:00+00:00, run_id=scheduled__2021-12-26T00:00:00+00:00, run_start_date=2022-05-15 01:30:09.130643+00:00, run_end_date=2022-05-15 01:30:48.884494+00:00, run_duration=39.753851, state=failed, external_trigger=False, run_type=scheduled, data_interval_start=2021-12-26 00:00:00+00:00, data_interval_end=2021-12-27 00:00:00+00:00, dag_hash=08ba76bc4de643e0df82526ee6b4f9f1 30 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2021-12-27T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:38,807] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:30:39,088] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:30:39,090] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:30:39,095] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:30:39,102] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:30:39,432] {taskinstance.py:1262} INFO - Executing on 2021-12-27 00:00:00+00:00 9 | [2022-05-15 01:30:40,597] {standard_task_runner.py:52} INFO - Started process 2595 to run task 10 | [2022-05-15 01:30:46,130] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2021-12-27T00:00:00+00:00', '--job-id', '50', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpp2km720p', '--error-file', '/tmp/tmppwbfv9md'] 11 | [2022-05-15 01:30:46,202] {standard_task_runner.py:77} INFO - Job 50: Subtask test 12 | [2022-05-15 01:30:48,667] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:30:49,787] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2021-12-27T00:00:00+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2021-12-27T00:00:00+00:00 19 | [2022-05-15 01:30:49,857] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:30:49,869] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:30:49,959] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:30:49,984] {subprocess.py:89} INFO - /tmp/***tmpu_3zle15 24 | [2022-05-15 01:30:49,990] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:30:50,134] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20211227T000000, start_date=20220515T013038, end_date=20220515T013050 26 | [2022-05-15 01:30:50,294] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:30:50,612] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 28 | [2022-05-15 01:30:50,686] {dagrun.py:527} ERROR - Marking run failed 29 | [2022-05-15 01:30:50,706] {dagrun.py:602} INFO - DagRun Finished: dag_id=dbt_dag, execution_date=2021-12-27 00:00:00+00:00, run_id=scheduled__2021-12-27T00:00:00+00:00, run_start_date=2022-05-15 01:30:09.838333+00:00, run_end_date=2022-05-15 01:30:50.706438+00:00, run_duration=40.868105, state=failed, external_trigger=False, run_type=scheduled, data_interval_start=2021-12-27 00:00:00+00:00, data_interval_end=2021-12-28 00:00:00+00:00, dag_hash=08ba76bc4de643e0df82526ee6b4f9f1 30 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2021-12-28T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:37,264] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:30:37,616] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:30:37,628] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:30:37,632] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:30:37,652] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:30:38,175] {taskinstance.py:1262} INFO - Executing on 2021-12-28 00:00:00+00:00 9 | [2022-05-15 01:30:38,323] {standard_task_runner.py:52} INFO - Started process 2592 to run task 10 | [2022-05-15 01:30:38,403] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2021-12-28T00:00:00+00:00', '--job-id', '47', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpkg4kc1dg', '--error-file', '/tmp/tmp5n8pb8cs'] 11 | [2022-05-15 01:30:38,439] {standard_task_runner.py:77} INFO - Job 47: Subtask test 12 | [2022-05-15 01:30:39,489] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:30:48,575] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2021-12-28T00:00:00+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2021-12-28T00:00:00+00:00 19 | [2022-05-15 01:30:48,645] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:30:48,648] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:30:48,760] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:30:48,767] {subprocess.py:89} INFO - /tmp/***tmphdt_h70u 24 | [2022-05-15 01:30:48,770] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:30:48,958] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20211228T000000, start_date=20220515T013037, end_date=20220515T013048 26 | [2022-05-15 01:30:49,291] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:30:49,935] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 28 | [2022-05-15 01:30:50,063] {dagrun.py:527} ERROR - Marking run failed 29 | [2022-05-15 01:30:50,066] {dagrun.py:602} INFO - DagRun Finished: dag_id=dbt_dag, execution_date=2021-12-28 00:00:00+00:00, run_id=scheduled__2021-12-28T00:00:00+00:00, run_start_date=2022-05-15 01:30:10.436572+00:00, run_end_date=2022-05-15 01:30:50.066189+00:00, run_duration=39.629617, state=failed, external_trigger=False, run_type=scheduled, data_interval_start=2021-12-28 00:00:00+00:00, data_interval_end=2021-12-29 00:00:00+00:00, dag_hash=08ba76bc4de643e0df82526ee6b4f9f1 30 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test/2021-12-30T00:00:00+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 01:30:31,916] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 01:30:33,866] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 01:30:33,871] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 01:30:33,883] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 01:30:33,924] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 01:30:34,348] {taskinstance.py:1262} INFO - Executing on 2021-12-30 00:00:00+00:00 9 | [2022-05-15 01:30:34,522] {standard_task_runner.py:52} INFO - Started process 2590 to run task 10 | [2022-05-15 01:30:34,790] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test', 'scheduled__2021-12-30T00:00:00+00:00', '--job-id', '46', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpi7g4nao5', '--error-file', '/tmp/tmp236welhr'] 11 | [2022-05-15 01:30:34,815] {standard_task_runner.py:77} INFO - Job 46: Subtask test 12 | [2022-05-15 01:30:36,979] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 01:30:38,841] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test 17 | AIRFLOW_CTX_EXECUTION_DATE=2021-12-30T00:00:00+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2021-12-30T00:00:00+00:00 19 | [2022-05-15 01:30:38,866] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 01:30:38,883] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n pwd\n '] 22 | [2022-05-15 01:30:39,177] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 01:30:39,318] {subprocess.py:89} INFO - /tmp/***tmps65exmi0 24 | [2022-05-15 01:30:39,352] {subprocess.py:93} INFO - Command exited with return code 0 25 | [2022-05-15 01:30:47,124] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test, execution_date=20211230T000000, start_date=20220515T013032, end_date=20220515T013047 26 | [2022-05-15 01:30:48,266] {local_task_job.py:154} INFO - Task exited with return code 0 27 | [2022-05-15 01:30:48,904] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 28 | [2022-05-15 01:30:49,190] {dagrun.py:527} ERROR - Marking run failed 29 | [2022-05-15 01:30:49,205] {dagrun.py:602} INFO - DagRun Finished: dag_id=dbt_dag, execution_date=2021-12-30 00:00:00+00:00, run_id=scheduled__2021-12-30T00:00:00+00:00, run_start_date=2022-05-15 01:30:12.114131+00:00, run_end_date=2022-05-15 01:30:49.204932+00:00, run_duration=37.090801, state=failed, external_trigger=False, run_type=scheduled, data_interval_start=2021-12-30 00:00:00+00:00, data_interval_end=2021-12-31 00:00:00+00:00, dag_hash=08ba76bc4de643e0df82526ee6b4f9f1 30 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.assert_positive_total_for_payments/2022-05-17T22:12:00.138285+00:00/4.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:36:02,141] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:36:02,287] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:36:02,297] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:36:02,300] {taskinstance.py:1242} INFO - Starting attempt 4 of 6 6 | [2022-05-18 23:36:02,321] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:36:03,088] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:36:03,621] {standard_task_runner.py:52} INFO - Started process 601 to run task 10 | [2022-05-18 23:36:03,681] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.assert_positive_total_for_payments', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '178', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmp1n7z6bq4', '--error-file', '/tmp/tmp97shrc6g'] 11 | [2022-05-18 23:36:03,701] {standard_task_runner.py:77} INFO - Job 178: Subtask dbt_test.test.jaffle_shop.assert_positive_total_for_payments 12 | [2022-05-18 23:36:05,150] {logging_mixin.py:109} INFO - Running on host d4b673e28191 13 | [2022-05-18 23:36:06,120] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.assert_positive_total_for_payments 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:36:06,123] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:36:06,131] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select assert_positive_total_for_payments --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:36:06,187] {subprocess.py:85} INFO - Output: 23 | [2022-05-18 23:37:40,013] {local_task_job.py:154} INFO - Task exited with return code Negsignal.SIGKILL 24 | [2022-05-18 23:38:09,067] {taskinstance.py:1280} INFO - Marking task as UP_FOR_RETRY. dag_id=model_level_dbt_great_expectations_dag, task_id=dbt_test.test.jaffle_shop.assert_positive_total_for_payments, execution_date=20220517T221200, start_date=20220518T233602, end_date=20220518T233808 25 | [2022-05-18 23:38:19,939] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 26 | -------------------------------------------------------------------------------- /superset/docker/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Getting Started with Superset using Docker 21 | 22 | Docker is an easy way to get started with Superset. 23 | 24 | ## Prerequisites 25 | 26 | 1. Docker! [link](https://www.docker.com/get-started) 27 | 2. Docker-compose [link](https://docs.docker.com/compose/install/) 28 | 29 | ## Configuration 30 | 31 | The `/app/pythonpath` folder is mounted from [`./docker/pythonpath_dev`](./pythonpath_dev) 32 | which contains a base configuration [`./docker/pythonpath_dev/superset_config.py`](./pythonpath_dev/superset_config.py) 33 | intended for use with local development. 34 | 35 | ### Local overrides 36 | 37 | In order to override configuration settings locally, simply make a copy of [`./docker/pythonpath_dev/superset_config_local.example`](./pythonpath_dev/superset_config_local.example) 38 | into `./docker/pythonpath_dev/superset_config_docker.py` (git ignored) and fill in your overrides. 39 | 40 | ### Local packages 41 | 42 | If you want to add Python packages in order to test things like databases locally, you can simply add a local requirements.txt (`./docker/requirements-local.txt`) 43 | and rebuild your Docker stack. 44 | 45 | Steps: 46 | 47 | 1. Create `./docker/requirements-local.txt` 48 | 2. Add your new packages 49 | 3. Rebuild docker-compose 50 | 1. `docker-compose down -v` 51 | 2. `docker-compose up` 52 | 53 | ## Initializing Database 54 | 55 | The database will initialize itself upon startup via the init container ([`superset-init`](./docker-init.sh)). This may take a minute. 56 | 57 | ## Normal Operation 58 | 59 | To run the container, simply run: `docker-compose up` 60 | 61 | After waiting several minutes for Superset initialization to finish, you can open a browser and view [`http://localhost:8088`](http://localhost:8088) 62 | to start your journey. 63 | 64 | ## Developing 65 | 66 | While running, the container server will reload on modification of the Superset Python and JavaScript source code. 67 | Don't forget to reload the page to take the new frontend into account though. 68 | 69 | ## Production 70 | 71 | It is possible to run Superset in non-development mode by using [`docker-compose-non-dev.yml`](../docker-compose-non-dev.yml). This file excludes the volumes needed for development and uses [`./docker/.env-non-dev`](./.env-non-dev) which sets the variable `SUPERSET_ENV` to `production`. 72 | 73 | ## Resource Constraints 74 | 75 | If you are attempting to build on macOS and it exits with 137 you need to increase your Docker resources. See instructions [here](https://docs.docker.com/docker-for-mac/#advanced) (search for memory) 76 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test.jaffle_shop.int_orders/2022-05-15T03:11:14.036092+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 03:11:33,634] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 03:11:33,660] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 03:11:33,661] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 03:11:33,662] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 03:11:33,663] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 03:11:33,688] {taskinstance.py:1262} INFO - Executing on 2022-05-15 03:11:14.036092+00:00 9 | [2022-05-15 03:11:33,694] {standard_task_runner.py:52} INFO - Started process 4293 to run task 10 | [2022-05-15 03:11:33,723] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test.jaffle_shop.int_orders', 'manual__2022-05-15T03:11:14.036092+00:00', '--job-id', '88', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpj4lwkpwv', '--error-file', '/tmp/tmpl6u3hngu'] 11 | [2022-05-15 03:11:33,726] {standard_task_runner.py:77} INFO - Job 88: Subtask test.jaffle_shop.int_orders 12 | [2022-05-15 03:11:33,815] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 03:11:33,937] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test.jaffle_shop.int_orders 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-15T03:11:14.036092+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=manual__2022-05-15T03:11:14.036092+00:00 19 | [2022-05-15 03:11:33,939] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 03:11:33,941] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n cd /opt/***/dags/jaffle_shop &&\n dbt --no-write-json test --target prod --models int_orders --profiles-dir .\n '] 22 | [2022-05-15 03:11:33,956] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 03:11:37,577] {subprocess.py:89} INFO - 03:11:37 Running with dbt=1.1.0 24 | [2022-05-15 03:11:37,702] {subprocess.py:89} INFO - 03:11:37 Found 5 models, 4 tests, 0 snapshots, 0 analyses, 170 macros, 0 operations, 0 seed files, 3 sources, 0 exposures, 0 metrics 25 | [2022-05-15 03:11:37,712] {subprocess.py:89} INFO - 03:11:37 26 | [2022-05-15 03:11:37,716] {subprocess.py:89} INFO - 03:11:37 [WARNING]: Nothing to do. Try checking your model configs and model specification args 27 | [2022-05-15 03:11:38,843] {subprocess.py:93} INFO - Command exited with return code 0 28 | [2022-05-15 03:11:38,874] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test.jaffle_shop.int_orders, execution_date=20220515T031114, start_date=20220515T031133, end_date=20220515T031138 29 | [2022-05-15 03:11:38,929] {local_task_job.py:154} INFO - Task exited with return code 0 30 | [2022-05-15 03:11:38,966] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 31 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.source_not_null_jaffle_shop_customers_id.50aa22178f/2022-05-17T22:12:00.138285+00:00/2.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:02:42,404] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:02:42,698] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:02:42,705] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:02:42,712] {taskinstance.py:1242} INFO - Starting attempt 2 of 5 6 | [2022-05-18 23:02:42,716] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:02:43,233] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:02:47,230] {standard_task_runner.py:52} INFO - Started process 430 to run task 10 | [2022-05-18 23:02:48,343] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.source_not_null_jaffle_shop_customers_id.50aa22178f', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '146', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmp47s5gcl6', '--error-file', '/tmp/tmp4qd0_5u_'] 11 | [2022-05-18 23:02:48,359] {standard_task_runner.py:77} INFO - Job 146: Subtask dbt_test.test.jaffle_shop.source_not_null_jaffle_shop_customers_id.50aa22178f 12 | [2022-05-18 23:02:49,725] {logging_mixin.py:109} INFO - Running on host 3d7cb1bfa13d 13 | [2022-05-18 23:03:09,407] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.source_not_null_jaffle_shop_customers_id.50aa22178f 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:03:09,626] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:03:09,651] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select source_not_null_jaffle_shop_customers_id --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:03:19,507] {subprocess.py:85} INFO - Output: 23 | [2022-05-18 23:03:25,635] {local_task_job.py:154} INFO - Task exited with return code Negsignal.SIGKILL 24 | [2022-05-18 23:03:29,134] {taskinstance.py:1280} INFO - Marking task as UP_FOR_RETRY. dag_id=model_level_dbt_great_expectations_dag, task_id=dbt_test.test.jaffle_shop.source_not_null_jaffle_shop_customers_id.50aa22178f, execution_date=20220517T221200, start_date=20220518T230242, end_date=20220518T230329 25 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.source_unique_stripe_payment_id.3b7989ce64/2022-05-17T22:12:00.138285+00:00/2.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:02:37,885] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:02:38,318] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:02:38,334] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:02:38,345] {taskinstance.py:1242} INFO - Starting attempt 2 of 5 6 | [2022-05-18 23:02:38,349] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:02:38,501] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:02:38,534] {standard_task_runner.py:52} INFO - Started process 407 to run task 10 | [2022-05-18 23:02:38,586] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.source_unique_stripe_payment_id.3b7989ce64', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '138', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmp1ugu3jyo', '--error-file', '/tmp/tmp6g6oeabd'] 11 | [2022-05-18 23:02:38,587] {standard_task_runner.py:77} INFO - Job 138: Subtask dbt_test.test.jaffle_shop.source_unique_stripe_payment_id.3b7989ce64 12 | [2022-05-18 23:02:39,352] {logging_mixin.py:109} INFO - Running on host 3d7cb1bfa13d 13 | [2022-05-18 23:02:39,846] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.source_unique_stripe_payment_id.3b7989ce64 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:02:39,848] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:02:39,850] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select source_unique_stripe_payment_id --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:02:40,142] {subprocess.py:85} INFO - Output: 23 | [2022-05-18 23:04:19,049] {local_task_job.py:154} INFO - Task exited with return code Negsignal.SIGKILL 24 | [2022-05-18 23:04:26,396] {taskinstance.py:1280} INFO - Marking task as UP_FOR_RETRY. dag_id=model_level_dbt_great_expectations_dag, task_id=dbt_test.test.jaffle_shop.source_unique_stripe_payment_id.3b7989ce64, execution_date=20220517T221200, start_date=20220518T230237, end_date=20220518T230426 25 | [2022-05-18 23:04:33,796] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 26 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c/2022-05-17T22:12:00.138285+00:00/2.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:02:38,523] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:02:38,601] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:02:38,604] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:02:38,606] {taskinstance.py:1242} INFO - Starting attempt 2 of 5 6 | [2022-05-18 23:02:38,608] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:02:38,954] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:02:39,072] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '140', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmpf4kq78e9', '--error-file', '/tmp/tmpljsydyo_'] 10 | [2022-05-18 23:02:39,067] {standard_task_runner.py:52} INFO - Started process 409 to run task 11 | [2022-05-18 23:02:39,080] {standard_task_runner.py:77} INFO - Job 140: Subtask dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c 12 | [2022-05-18 23:02:39,584] {logging_mixin.py:109} INFO - Running on host 3d7cb1bfa13d 13 | [2022-05-18 23:02:40,500] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:02:40,507] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:02:40,511] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select source_not_null_stripe_payment_id --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:02:40,642] {subprocess.py:85} INFO - Output: 23 | [2022-05-18 23:04:19,048] {local_task_job.py:154} INFO - Task exited with return code Negsignal.SIGKILL 24 | [2022-05-18 23:04:26,125] {taskinstance.py:1280} INFO - Marking task as UP_FOR_RETRY. dag_id=model_level_dbt_great_expectations_dag, task_id=dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c, execution_date=20220517T221200, start_date=20220518T230238, end_date=20220518T230425 25 | [2022-05-18 23:04:30,330] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 26 | -------------------------------------------------------------------------------- /airflow/logs/model_level_dbt_great_expectations_dag/dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c/2022-05-17T22:12:00.138285+00:00/5.log: -------------------------------------------------------------------------------- 1 | [2022-05-18 23:35:27,409] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-18 23:35:27,521] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-18 23:35:27,527] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-18 23:35:27,529] {taskinstance.py:1242} INFO - Starting attempt 5 of 7 6 | [2022-05-18 23:35:27,534] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-18 23:35:27,787] {taskinstance.py:1262} INFO - Executing on 2022-05-17 22:12:00.138285+00:00 9 | [2022-05-18 23:35:27,921] {standard_task_runner.py:52} INFO - Started process 577 to run task 10 | [2022-05-18 23:35:28,132] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'model_level_dbt_great_expectations_dag', 'dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c', 'scheduled__2022-05-17T22:12:00.138285+00:00', '--job-id', '173', '--raw', '--subdir', 'DAGS_FOLDER/model_level_dbt_great_expectations_dag.py', '--cfg-path', '/tmp/tmpz_u3yw1g', '--error-file', '/tmp/tmp43p_5got'] 11 | [2022-05-18 23:35:28,177] {standard_task_runner.py:77} INFO - Job 173: Subtask dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c 12 | [2022-05-18 23:35:38,033] {logging_mixin.py:109} INFO - Running on host d4b673e28191 13 | [2022-05-18 23:35:47,475] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=luciano.naveiro 15 | AIRFLOW_CTX_DAG_ID=model_level_dbt_great_expectations_dag 16 | AIRFLOW_CTX_TASK_ID=dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-17T22:12:00.138285+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=scheduled__2022-05-17T22:12:00.138285+00:00 19 | [2022-05-18 23:35:49,783] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-18 23:35:50,195] {subprocess.py:74} INFO - Running command: ['bash', '-c', 'dbt --no-write-json test --target dev --select source_not_null_stripe_payment_id --profiles-dir /opt/dbt/jaffle_shop --project-dir /opt/dbt/jaffle_shop'] 22 | [2022-05-18 23:36:01,587] {subprocess.py:85} INFO - Output: 23 | [2022-05-18 23:37:53,647] {local_task_job.py:154} INFO - Task exited with return code Negsignal.SIGKILL 24 | [2022-05-18 23:38:12,497] {taskinstance.py:1280} INFO - Marking task as UP_FOR_RETRY. dag_id=model_level_dbt_great_expectations_dag, task_id=dbt_test.test.jaffle_shop.source_not_null_stripe_payment_id.f33f317f6c, execution_date=20220517T221200, start_date=20220518T233527, end_date=20220518T233811 25 | [2022-05-18 23:38:22,463] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 26 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test.jaffle_shop.fct_customer_orders/2022-05-15T03:11:14.036092+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 03:11:40,530] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 03:11:40,545] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 03:11:40,547] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 03:11:40,548] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 03:11:40,548] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 03:11:40,563] {taskinstance.py:1262} INFO - Executing on 2022-05-15 03:11:14.036092+00:00 9 | [2022-05-15 03:11:40,568] {standard_task_runner.py:52} INFO - Started process 4311 to run task 10 | [2022-05-15 03:11:40,572] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test.jaffle_shop.fct_customer_orders', 'manual__2022-05-15T03:11:14.036092+00:00', '--job-id', '89', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmpn9sdblhd', '--error-file', '/tmp/tmp2c617di9'] 11 | [2022-05-15 03:11:40,574] {standard_task_runner.py:77} INFO - Job 89: Subtask test.jaffle_shop.fct_customer_orders 12 | [2022-05-15 03:11:40,627] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 03:11:40,686] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test.jaffle_shop.fct_customer_orders 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-15T03:11:14.036092+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=manual__2022-05-15T03:11:14.036092+00:00 19 | [2022-05-15 03:11:40,688] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 03:11:40,689] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n cd /opt/***/dags/jaffle_shop &&\n dbt --no-write-json test --target prod --models fct_customer_orders --profiles-dir .\n '] 22 | [2022-05-15 03:11:40,702] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 03:11:44,159] {subprocess.py:89} INFO - 03:11:44 Running with dbt=1.1.0 24 | [2022-05-15 03:11:44,313] {subprocess.py:89} INFO - 03:11:44 Found 5 models, 4 tests, 0 snapshots, 0 analyses, 170 macros, 0 operations, 0 seed files, 3 sources, 0 exposures, 0 metrics 25 | [2022-05-15 03:11:44,330] {subprocess.py:89} INFO - 03:11:44 26 | [2022-05-15 03:11:44,335] {subprocess.py:89} INFO - 03:11:44 [WARNING]: Nothing to do. Try checking your model configs and model specification args 27 | [2022-05-15 03:11:45,443] {subprocess.py:93} INFO - Command exited with return code 0 28 | [2022-05-15 03:11:45,478] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test.jaffle_shop.fct_customer_orders, execution_date=20220515T031114, start_date=20220515T031140, end_date=20220515T031145 29 | [2022-05-15 03:11:45,548] {local_task_job.py:154} INFO - Task exited with return code 0 30 | [2022-05-15 03:11:45,585] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 31 | -------------------------------------------------------------------------------- /airflow/logs/dbt_dag/test.jaffle_shop.stg_stripe__payments/2022-05-15T03:11:14.036092+00:00/1.log: -------------------------------------------------------------------------------- 1 | [2022-05-15 03:11:26,511] {taskinstance.py:1035} INFO - Dependencies all met for 2 | [2022-05-15 03:11:26,544] {taskinstance.py:1035} INFO - Dependencies all met for 3 | [2022-05-15 03:11:26,545] {taskinstance.py:1241} INFO - 4 | -------------------------------------------------------------------------------- 5 | [2022-05-15 03:11:26,547] {taskinstance.py:1242} INFO - Starting attempt 1 of 1 6 | [2022-05-15 03:11:26,548] {taskinstance.py:1243} INFO - 7 | -------------------------------------------------------------------------------- 8 | [2022-05-15 03:11:26,586] {taskinstance.py:1262} INFO - Executing on 2022-05-15 03:11:14.036092+00:00 9 | [2022-05-15 03:11:26,601] {standard_task_runner.py:52} INFO - Started process 4258 to run task 10 | [2022-05-15 03:11:26,620] {standard_task_runner.py:76} INFO - Running: ['***', 'tasks', 'run', 'dbt_dag', 'test.jaffle_shop.stg_stripe__payments', 'manual__2022-05-15T03:11:14.036092+00:00', '--job-id', '86', '--raw', '--subdir', 'DAGS_FOLDER/dbt_dag.py', '--cfg-path', '/tmp/tmp2h1zbost', '--error-file', '/tmp/tmpubfiyk4f'] 11 | [2022-05-15 03:11:26,622] {standard_task_runner.py:77} INFO - Job 86: Subtask test.jaffle_shop.stg_stripe__payments 12 | [2022-05-15 03:11:26,705] {logging_mixin.py:109} INFO - Running on host b0833397c3cf 13 | [2022-05-15 03:11:26,829] {taskinstance.py:1429} INFO - Exporting the following env vars: 14 | AIRFLOW_CTX_DAG_OWNER=*** 15 | AIRFLOW_CTX_DAG_ID=dbt_dag 16 | AIRFLOW_CTX_TASK_ID=test.jaffle_shop.stg_stripe__payments 17 | AIRFLOW_CTX_EXECUTION_DATE=2022-05-15T03:11:14.036092+00:00 18 | AIRFLOW_CTX_DAG_RUN_ID=manual__2022-05-15T03:11:14.036092+00:00 19 | [2022-05-15 03:11:26,835] {subprocess.py:62} INFO - Tmp dir root location: 20 | /tmp 21 | [2022-05-15 03:11:26,838] {subprocess.py:74} INFO - Running command: ['bash', '-c', '\n cd /opt/***/dags/jaffle_shop &&\n dbt --no-write-json test --target prod --models stg_stripe__payments --profiles-dir .\n '] 22 | [2022-05-15 03:11:26,862] {subprocess.py:85} INFO - Output: 23 | [2022-05-15 03:11:32,210] {subprocess.py:89} INFO - 03:11:32 Running with dbt=1.1.0 24 | [2022-05-15 03:11:32,353] {subprocess.py:89} INFO - 03:11:32 Found 5 models, 4 tests, 0 snapshots, 0 analyses, 170 macros, 0 operations, 0 seed files, 3 sources, 0 exposures, 0 metrics 25 | [2022-05-15 03:11:32,364] {subprocess.py:89} INFO - 03:11:32 26 | [2022-05-15 03:11:32,367] {subprocess.py:89} INFO - 03:11:32 [WARNING]: Nothing to do. Try checking your model configs and model specification args 27 | [2022-05-15 03:11:33,697] {subprocess.py:93} INFO - Command exited with return code 0 28 | [2022-05-15 03:11:33,736] {taskinstance.py:1280} INFO - Marking task as SUCCESS. dag_id=dbt_dag, task_id=test.jaffle_shop.stg_stripe__payments, execution_date=20220515T031114, start_date=20220515T031126, end_date=20220515T031133 29 | [2022-05-15 03:11:33,794] {local_task_job.py:154} INFO - Task exited with return code 0 30 | [2022-05-15 03:11:33,856] {local_task_job.py:264} INFO - 0 downstream tasks scheduled from follow-on schedule check 31 | --------------------------------------------------------------------------------