├── integration_tests ├── macros │ ├── .gitkeep │ ├── get_nodes_testing.sql │ └── notes.sql ├── seeds │ ├── .gitkeep │ ├── expected │ │ ├── ratio_metric__expected.csv │ │ ├── base_average_metric__expected.csv │ │ ├── base_count_metric__no_start_date_expected.csv │ │ ├── base_count_metric__no_end_date_expected.csv │ │ └── base_count_metric__secondary_calculations_expected.csv │ └── source │ │ ├── seed_slack_users.csv │ │ ├── mock_purchase_data.csv │ │ ├── dim_customers_source.csv │ │ ├── fact_orders_source.csv │ │ └── fact_orders_duplicate_source.csv ├── tests │ └── .gitkeep ├── models │ ├── metric_definitions │ │ ├── expression_metric.yml │ │ ├── base_count_metric.yml │ │ ├── case_when_metric.yml │ │ ├── base_average_metric.yml │ │ ├── ratio_metric.yml │ │ ├── base_median_metric.yml │ │ ├── derived_metric__alternative.yml │ │ ├── metric_on_derived_metric.yml │ │ ├── derived_metric.yml │ │ ├── base_count_distinct_metric.yml │ │ └── base_sum_metric.yml │ ├── materialized_models │ │ ├── dim_customers.sql │ │ ├── fact_orders.sql │ │ ├── fact_orders_duplicate.sql │ │ ├── combined__orders_customers.sql │ │ ├── dim_customers.yml │ │ └── fact_orders.yml │ ├── metric_testing_models │ │ ├── derived_metric.yml │ │ ├── base_sum_metric.yml │ │ ├── base_average_metric.yml │ │ ├── base_sum_metric.sql │ │ ├── metric_on_derived_metric.yml │ │ ├── multiple_metrics__rolling.yml │ │ ├── ratio_metric.sql │ │ ├── base_count_metric__no_end_date.yml │ │ ├── multiple_metrics__base_metrics.yml │ │ ├── base_count_metric__no_start_date.yml │ │ ├── multiple_metrics__period_to_date.yml │ │ ├── multiple_metrics__period_over_period.yml │ │ ├── base_median_metric_no_time_grain.sql │ │ ├── base_count_metric__secondary_calculations.yml │ │ ├── testing_metrics.sql │ │ ├── base_average_metric__all_time.sql │ │ ├── base_count_distinct_metric.sql │ │ ├── case_when_metric.sql │ │ ├── base_average_metric.sql │ │ ├── base_no_timestamp_metric.sql │ │ ├── base_sum_metric__14_day_window.sql │ │ ├── base_count_metric__no_end_date.sql │ │ ├── base_count_metric__no_start_date.sql │ │ ├── derived_metric__no_dimensions.sql │ │ ├── derived_metric__secondary_calculations.sql │ │ ├── multiple_metrics__base_metrics.sql │ │ ├── multiple_metrics__derived_metrics.sql │ │ ├── derived_metric.sql │ │ ├── base_median_metric.sql │ │ ├── base_sum_metric__prior.sql │ │ ├── metric_on_derived_metric.sql │ │ ├── multiple_metrics__period_to_date.sql │ │ ├── multiple_metrics__rolling.sql │ │ ├── multiple_metrics__period_over_period.sql │ │ ├── develop_metric_no_timestamp.sql │ │ ├── simple_develop_metric.sql │ │ ├── base_count_metric__secondary_calculations.sql │ │ └── develop_metric.sql │ └── custom_calendar.sql ├── .gitignore ├── packages.yml ├── README.md └── dbt_project.yml ├── CODEOWNERS ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── feature-request.yml │ ├── regression-report.yml │ └── bug-report.yml ├── workflows │ ├── jira-creation.yml │ ├── jira-label.yml │ ├── jira-transition.yml │ ├── triage-labels.yml │ ├── stale.yml │ ├── changelog-existence.yml │ └── bot-changelog.yml ├── actions │ └── end-to-end-test │ │ └── action.yml └── pull_request_template.md ├── .changes ├── 0.0.0.md ├── unreleased │ ├── Docs-20221010-144621.yaml │ ├── Features-20230109-145530.yaml │ ├── Fixes-20221117-113242.yaml │ ├── Fixes-20221127-161855.yaml │ ├── Features-20221128-124335.yaml │ ├── Fixes-20221025-151422.yaml │ ├── Fixes-20221116-145448.yaml │ ├── Fixes-20230106-110532.yaml │ ├── Features-20221117-162500.yaml │ ├── Features-20230103-152219.yaml │ ├── Features-20230111-105620.yaml │ ├── Fixes-20221025-164009.yaml │ ├── Fixes-20230302-090511.yaml │ ├── Docs-20221011-094243.yaml │ ├── Under the Hood-20221012-101830.yaml │ ├── Under the Hood-20221012-114117.yaml │ ├── Under the Hood-20230118-161610.yaml │ ├── Docs-20221025-170623.yaml │ ├── Breaking Changes-20230426-163623.yaml │ ├── Under the Hood-20230117-092325.yaml │ ├── Under the Hood-20230130-122730.yaml │ ├── Fixes-20221102-232947.yaml │ ├── Docs-20221104-142157.yaml │ └── Features-20221122-135016.yaml └── header.tpl.md ├── .gitignore ├── macros ├── validation │ ├── validate_calendar_model.sql │ ├── is_valid_dimension.sql │ ├── validate_where.sql │ ├── validate_aggregate_coherence.sql │ ├── validate_secondary_calculations.sql │ ├── validate_timestamp.sql │ ├── validate_grain_order.sql │ ├── validate_derived_metrics.sql │ ├── validate_develop_metrics.sql │ ├── validate_metric_config.sql │ ├── validate_dimension_list.sql │ └── validate_grain.sql ├── variables │ ├── get_metric_model_name.sql │ ├── get_grain_order.sql │ ├── get_relevent_periods.sql │ ├── get_non_calendar_dimension_list.sql │ ├── get_calendar_dimensions.sql │ ├── get_metric_allowlist.sql │ ├── get_total_dimension_count.sql │ ├── get_metrics_dictionary.sql │ ├── get_develop_unique_metric_id_list.sql │ ├── get_metric_list.sql │ ├── get_metric_unique_id_list.sql │ ├── get_base_metrics.sql │ ├── get_faux_metric_tree.sql │ ├── get_metric_definition.sql │ ├── get_models_grouping.sql │ ├── get_model_group.sql │ └── get_metric_tree.sql ├── graph_parsing │ ├── get_metric_relation.sql │ └── get_model_relation.sql ├── sql_gen │ ├── gen_order_by.sql │ ├── gen_dimensions_cte.sql │ ├── gen_calendar_cte.sql │ ├── gen_secondary_calculations.sql │ ├── gen_spine_time_cte.sql │ ├── gen_filters.sql │ ├── gen_group_by.sql │ ├── build_metric_sql.sql │ ├── gen_calendar_join.sql │ ├── gen_base_query.sql │ ├── gen_primary_metric_aggregate.sql │ ├── gen_property_to_aggregate.sql │ └── gen_metric_cte.sql ├── secondary_calculations │ ├── secondary_calculation_prior.sql │ ├── secondary_calculation_period_to_date.sql │ ├── secondary_calculation_rolling.sql │ ├── perform_secondary_calculation.sql │ ├── secondary_calculation_period_over_period.sql │ └── generate_secondary_calculation_alias.sql ├── secondary_calculations_configuration │ ├── prior.sql │ ├── rolling.sql │ ├── period_to_date.sql │ └── period_over_period.sql ├── misc │ ├── metrics__date_spine.sql │ └── metrics__equality.sql ├── README.md └── calculate.sql ├── pytest.ini ├── docker-compose.yml ├── models ├── dbt_metrics_default_calendar.yml └── dbt_metrics_default_calendar.sql ├── examples └── metric_jsonschema_example.json ├── dev-requirements.txt ├── dbt_project.yml ├── test.env.example ├── CHANGELOG.MD └── tests ├── functional ├── invalid_configs │ ├── test_invalid_develop_config__missing_timestamp.py │ ├── test_invalid_develop_config__invalid_model.py │ ├── test_invalid_develop_config__invalid_type.py │ ├── test_invalid_develop_config_invalid_model.py │ ├── test_invalid_date_datatype.py │ ├── test_invalid_develop_config__invalid_dimension.py │ ├── test_undefined_metric.py │ ├── test_invalid_metric_name.py │ ├── test_invalid_derived_metric.py │ ├── test_invalid_string_datatype.py │ ├── test_invalid_no_time_grain_secondary_calc.py │ ├── test_invalid_backwards_compatability_metric_list.py │ ├── test_invalid_period_to_date_average.py │ ├── test_invalid_backwards_compatability_expression_metric.py │ ├── test_invalid_ephemeral_model.py │ ├── test_invalid_derived_metric_filter.py │ ├── test_invalid_no_time_grain_calendar_dimension.py │ └── test_invalid_where.py └── metric_options │ └── old_metric_spec │ └── test_old_metric_spec.py └── conftest.py /integration_tests/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /integration_tests/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /integration_tests/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | @callum-mcdata 2 | @dave-connors-3 3 | @joellabes -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/expression_metric.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Owning team for this repo 2 | * @dbt-labs/dbt-package-owners 3 | -------------------------------------------------------------------------------- /integration_tests/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | examples/ 6 | model_testing/ -------------------------------------------------------------------------------- /integration_tests/models/materialized_models/dim_customers.sql: -------------------------------------------------------------------------------- 1 | select * from {{ref('dim_customers_source')}} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/derived_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: derived_metric -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_sum_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: base_sum_metric -------------------------------------------------------------------------------- /.changes/0.0.0.md: -------------------------------------------------------------------------------- 1 | ## Previous Releases 2 | 3 | For information on prior major and minor releases, see their changelogs: 4 | -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_average_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: base_average_metric -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_sum_metric.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from {{ metrics.calculate(metric('base_sum_metric'))}} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/metric_on_derived_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: metric_on_derived_metric -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/multiple_metrics__rolling.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: multiple_metrics__rolling -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/ratio_metric.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate(metric('ratio_metric') 4 | ) 5 | }} -------------------------------------------------------------------------------- /integration_tests/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - local: ../ 3 | - package: calogica/dbt_expectations 4 | version: [">=0.6.0", "<0.7.0"] 5 | -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_count_metric__no_end_date.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: base_count_metric__no_end_date -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/multiple_metrics__base_metrics.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: multiple_metrics__base_metrics -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_count_metric__no_start_date.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: base_count_metric__no_start_date -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/multiple_metrics__period_to_date.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: multiple_metrics__period_to_date -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/multiple_metrics__period_over_period.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: multiple_metrics__period_over_period -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_median_metric_no_time_grain.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate(metric('base_median_metric')) 4 | }} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_count_metric__secondary_calculations.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: base_count_metric__secondary_calculations 4 | -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/testing_metrics.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate(metric('base_count_metric'), 4 | grain='week' 5 | ) 6 | }} -------------------------------------------------------------------------------- /integration_tests/models/materialized_models/fact_orders.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | ,round(order_total - (order_total/2)) as discount_total 4 | from {{ref('fact_orders_source')}} -------------------------------------------------------------------------------- /.changes/unreleased/Docs-20221010-144621.yaml: -------------------------------------------------------------------------------- 1 | kind: Docs 2 | body: Adding changie 3 | time: 2022-10-10T14:46:21.414438-05:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "100" 7 | PR: "100" 8 | -------------------------------------------------------------------------------- /integration_tests/models/materialized_models/fact_orders_duplicate.sql: -------------------------------------------------------------------------------- 1 | select 2 | * 3 | ,round(order_total - (order_total/2)) as discount_total 4 | from {{ref('fact_orders_duplicate_source')}} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_average_metric__all_time.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate(metric('base_average_metric'), 4 | dimensions=['had_discount']) 5 | }} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_count_distinct_metric.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate(metric('base_count_distinct_metric'), 4 | grain='month' 5 | ) 6 | }} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/case_when_metric.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from {{ metrics.calculate( 3 | metric('case_when_metric'), 4 | grain='day' 5 | ) 6 | }} -------------------------------------------------------------------------------- /.changes/unreleased/Features-20230109-145530.yaml: -------------------------------------------------------------------------------- 1 | kind: Features 2 | body: Adding median 3 | time: 2023-01-09T14:55:30.09271-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "180" 7 | PR: "208" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Fixes-20221117-113242.yaml: -------------------------------------------------------------------------------- 1 | kind: Fixes 2 | body: Fixing date_day bug 3 | time: 2022-11-17T11:32:42.297464-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "184" 7 | PR: "185" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Fixes-20221127-161855.yaml: -------------------------------------------------------------------------------- 1 | kind: Fixes 2 | body: Hotfixing Order By 3 | time: 2022-11-27T16:18:55.910589-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "190" 7 | PR: "191" 8 | -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_average_metric.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate(metric('base_average_metric'), 4 | grain='day', 5 | dimensions=['had_discount']) 6 | }} -------------------------------------------------------------------------------- /.changes/unreleased/Features-20221128-124335.yaml: -------------------------------------------------------------------------------- 1 | kind: Features 2 | body: Databricks support 3 | time: 2022-11-28T12:43:35.333259-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "161" 7 | PR: "192" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Fixes-20221025-151422.yaml: -------------------------------------------------------------------------------- 1 | kind: Fixes 2 | body: fix typo in prior macro 3 | time: 2022-10-25T15:14:22.043378-04:00 4 | custom: 5 | Author: deanna-minnick 6 | Issue: "160" 7 | PR: "159" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Fixes-20221116-145448.yaml: -------------------------------------------------------------------------------- 1 | kind: Fixes 2 | body: Fixing calendar table join 3 | time: 2022-11-16T14:54:48.473292-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "181" 7 | PR: "180" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Fixes-20230106-110532.yaml: -------------------------------------------------------------------------------- 1 | kind: Fixes 2 | body: hotfixing variable name 3 | time: 2023-01-06T11:05:32.740588-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "207" 7 | PR: "207" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Features-20221117-162500.yaml: -------------------------------------------------------------------------------- 1 | kind: Features 2 | body: Making rolling unbound 3 | time: 2022-11-17T16:25:00.850213-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "140" 7 | PR: "186" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Features-20230103-152219.yaml: -------------------------------------------------------------------------------- 1 | kind: Features 2 | body: Making timestamps optional 3 | time: 2023-01-03T15:22:19.38878-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "200" 7 | PR: "201" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Features-20230111-105620.yaml: -------------------------------------------------------------------------------- 1 | kind: Features 2 | body: Adding date alias input 3 | time: 2023-01-11T10:56:20.26395-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "194" 7 | PR: "209" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Fixes-20221025-164009.yaml: -------------------------------------------------------------------------------- 1 | kind: Fixes 2 | body: Fixing bug with window functionality 3 | time: 2022-10-25T16:40:09.927002+01:00 4 | custom: 5 | Author: JoeryV 6 | Issue: "146" 7 | PR: "149" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Fixes-20230302-090511.yaml: -------------------------------------------------------------------------------- 1 | kind: Fixes 2 | body: fixes ephemeral model problem 3 | time: 2023-03-02T09:05:11.177408-08:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "222" 7 | PR: "222" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Docs-20221011-094243.yaml: -------------------------------------------------------------------------------- 1 | kind: Docs 2 | body: Fixing typo in rolling secondary calc 3 | time: 2022-10-11T09:42:43.49833-05:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "140" 7 | PR: "141" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Under the Hood-20221012-101830.yaml: -------------------------------------------------------------------------------- 1 | kind: Under the Hood 2 | body: Removing bool or 3 | time: 2022-10-12T10:18:30.482027-05:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "142" 7 | PR: "142" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Under the Hood-20221012-114117.yaml: -------------------------------------------------------------------------------- 1 | kind: Under the Hood 2 | body: Removing dbt_utils 3 | time: 2022-10-12T11:41:17.757328-05:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "143" 7 | PR: "143" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Under the Hood-20230118-161610.yaml: -------------------------------------------------------------------------------- 1 | kind: Under the Hood 2 | body: Updating github ci 3 | time: 2023-01-18T16:16:10.536918-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "163" 7 | PR: "213" 8 | -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_no_timestamp_metric.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | [metric('base_sum_metric__no_timestamp')] 5 | ,dimensions=['had_discount'] 6 | ) 7 | }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_modules/ 4 | dbt_packages/ 5 | logs/ 6 | .DS_Store 7 | integration_tests/target 8 | model_testing/ 9 | test.env 10 | env/ 11 | .venv 12 | venv/ 13 | .env 14 | __pycache__ 15 | .pytest_cache -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_sum_metric__14_day_window.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | metric('base_sum_metric__14_day_window'), 5 | grain='week', 6 | dimensions=[]) 7 | }} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_count_metric__no_end_date.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | metric('base_count_metric'), 5 | grain='month', 6 | start_date = '2021-02-01' 7 | ) 8 | }} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_count_metric__no_start_date.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | metric('base_count_metric'), 5 | grain='month', 6 | end_date = '2021-03-01' 7 | ) 8 | }} -------------------------------------------------------------------------------- /.changes/unreleased/Docs-20221025-170623.yaml: -------------------------------------------------------------------------------- 1 | kind: Docs 2 | body: Add changie as a PR condition to the CONTRIBUTING.md file 3 | time: 2022-10-25T17:06:23.469012+01:00 4 | custom: 5 | Author: JoeryV 6 | Issue: "154" 7 | PR: "155" 8 | -------------------------------------------------------------------------------- /macros/validation/validate_calendar_model.sql: -------------------------------------------------------------------------------- 1 | {% macro validate_calendar_model() %} 2 | 3 | {% set calendar_relation = metrics.get_model_relation(var('dbt_metrics_calendar_model', "dbt_metrics_default_calendar"))%} 4 | 5 | {% endmacro %} -------------------------------------------------------------------------------- /.changes/unreleased/Breaking Changes-20230426-163623.yaml: -------------------------------------------------------------------------------- 1 | kind: Breaking Changes 2 | body: Adding support for dbt-core 1.5 3 | time: 2023-04-26T16:36:23.710755-05:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "239" 7 | PR: "240" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Under the Hood-20230117-092325.yaml: -------------------------------------------------------------------------------- 1 | kind: Under the Hood 2 | body: Adding grouping for query generation 3 | time: 2023-01-17T09:23:25.796327-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "114" 7 | PR: "211" 8 | -------------------------------------------------------------------------------- /.changes/unreleased/Under the Hood-20230130-122730.yaml: -------------------------------------------------------------------------------- 1 | kind: Under the Hood 2 | body: validation logic on derived filters 3 | time: 2023-01-30T12:27:30.231612-06:00 4 | custom: 5 | Author: callum-mcdata 6 | Issue: "200" 7 | PR: "201" 8 | -------------------------------------------------------------------------------- /integration_tests/seeds/expected/ratio_metric__expected.csv: -------------------------------------------------------------------------------- 1 | date_month,base_sum_metric,base_average_metric,ratio_metric 2 | 2022-02-01,2,1.00000000000000000000,2.00000000000000000000 3 | 2022-01-01,8,1.00000000000000000000,8.00000000000000000000 -------------------------------------------------------------------------------- /integration_tests/seeds/source/seed_slack_users.csv: -------------------------------------------------------------------------------- 1 | user_id,joined_at,is_active_past_quarter,has_messaged 2 | 1,2021-01-01 14:18:27,true,true 3 | 2,2021-02-03 17:18:55,false,true 4 | 3,2021-04-01 11:01:28,false,false 5 | 4,2021-04-08 22:43:09,false,false -------------------------------------------------------------------------------- /macros/variables/get_metric_model_name.sql: -------------------------------------------------------------------------------- 1 | {% macro get_metric_model_name(metric_model) %} 2 | 3 | {% set metric_model_name = metric_model.replace('"','\'').split('\'')[1] %} 4 | 5 | {% do return(metric_model_name) %} 6 | 7 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/derived_metric__no_dimensions.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | metric('derived_metric'), 5 | grain='day', 6 | start_date = '2022-01-01', 7 | end_date = '2022-01-10') 8 | }} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/derived_metric__secondary_calculations.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | metric('ratio_metric'), 5 | grain='day', 6 | dimensions=['had_discount'] 7 | ) 8 | }} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/multiple_metrics__base_metrics.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | [metric('base_sum_metric'), metric('base_average_metric')], 5 | grain='day', 6 | dimensions=['had_discount']) 7 | }} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/multiple_metrics__derived_metrics.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | [metric('derived_metric'), metric('metric_on_derived_metric')], 5 | grain='day', 6 | dimensions=['had_discount']) 7 | }} -------------------------------------------------------------------------------- /.changes/unreleased/Fixes-20221102-232947.yaml: -------------------------------------------------------------------------------- 1 | kind: Fixes 2 | body: Updated the example metric_json_schema.yml file to reference the new properties. 3 | time: 2022-11-02T23:29:47.135949+02:00 4 | custom: 5 | Author: rijnhardtkotze 6 | Issue: "176" 7 | PR: "177" 8 | -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/derived_metric.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | [metric('derived_metric'),metric('base_count_distinct_metric')], 5 | grain='day', 6 | dimensions=['had_discount','order_country'] 7 | ) 8 | }} -------------------------------------------------------------------------------- /integration_tests/seeds/source/mock_purchase_data.csv: -------------------------------------------------------------------------------- 1 | purchased_at,payment_type,payment_total 2 | 2021-02-14 17:52:36,maestro,10 3 | 2021-02-15 04:16:50,jcb,10 4 | 2021-02-15 11:30:45,solo,10 5 | 2021-02-16 13:08:18,americanexpress,10 6 | 2021-02-17 05:41:34,americanexpress,10 -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_median_metric.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | [metric('base_median_metric'),metric('base_average_metric')], 5 | grain='month', 6 | dimensions=['had_discount'], 7 | date_alias='dat') 8 | }} -------------------------------------------------------------------------------- /integration_tests/seeds/expected/base_average_metric__expected.csv: -------------------------------------------------------------------------------- 1 | date_month,had_discount,base_average_metric 2 | 2022-01-01,TRUE,1.00000000000000000000 3 | 2022-01-01,FALSE,1.00000000000000000000 4 | 2022-02-01,FALSE,1.00000000000000000000 5 | 2022-02-01,TRUE,1.00000000000000000000 -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning 4 | ignore:unclosed file .*:ResourceWarning 5 | env_override_existing_values = 1 6 | env_files = 7 | test.env 8 | testpaths = 9 | tests/functional -------------------------------------------------------------------------------- /.changes/unreleased/Docs-20221104-142157.yaml: -------------------------------------------------------------------------------- 1 | kind: Docs 2 | body: Updated the example test environment file to match what is required across all 3 | platforms 4 | time: 2022-11-04T14:21:57.704467+02:00 5 | custom: 6 | Author: rijnhardtkotze 7 | Issue: "178" 8 | PR: "179" 9 | -------------------------------------------------------------------------------- /.changes/unreleased/Features-20221122-135016.yaml: -------------------------------------------------------------------------------- 1 | kind: Features 2 | body: DRYer code in gen_final_cte macro. Moved relevant portions into gen_group_by 3 | macro. 4 | time: 2022-11-22T13:50:16.559433-05:00 5 | custom: 6 | Author: deanna-minnick 7 | Issue: "144" 8 | PR: "189" 9 | -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_sum_metric__prior.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | metric('base_sum_metric'), 5 | grain='week', 6 | dimensions=['had_discount'], 7 | secondary_calculations=[metrics.prior(interval=3)] 8 | ) 9 | }} -------------------------------------------------------------------------------- /macros/variables/get_grain_order.sql: -------------------------------------------------------------------------------- 1 | {% macro get_grain_order() %} 2 | {{ return(adapter.dispatch('get_grain_order', 'metrics')()) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_grain_order() %} 6 | {% do return (['day', 'week', 'month', 'quarter', 'year']) %} 7 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/seeds/expected/base_count_metric__no_start_date_expected.csv: -------------------------------------------------------------------------------- 1 | date_month,has_messaged,is_active_past_quarter,base_count_metric 2 | 2021-01-01,TRUE,TRUE,1 3 | 2021-02-01,TRUE,FALSE,1 4 | 2021-03-01,TRUE,FALSE,0 5 | 2021-02-01,TRUE,TRUE,0 6 | 2021-03-01,TRUE,TRUE,0 7 | 2021-01-01,TRUE,FALSE,0 -------------------------------------------------------------------------------- /integration_tests/seeds/expected/base_count_metric__no_end_date_expected.csv: -------------------------------------------------------------------------------- 1 | date_month,has_messaged,is_active_past_quarter,base_count_metric 2 | 2021-02-01,TRUE,FALSE,1 3 | 2021-04-01,FALSE,FALSE,2 4 | 2021-03-01,TRUE,FALSE,0 5 | 2021-03-01,FALSE,FALSE,0 6 | 2021-02-01,FALSE,FALSE,0 7 | 2021-04-01,TRUE,FALSE,0 -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.5" 2 | services: 3 | database: 4 | image: postgres 5 | environment: 6 | POSTGRES_USER: "root" 7 | POSTGRES_PASSWORD: "password" 8 | POSTGRES_DB: "dbt" 9 | ports: 10 | - "5432:5432" 11 | 12 | networks: 13 | default: 14 | name: dbt-net -------------------------------------------------------------------------------- /macros/graph_parsing/get_metric_relation.sql: -------------------------------------------------------------------------------- 1 | {% macro get_metric_relation(ref_name) %} 2 | 3 | {% if execute %} 4 | {% set relation = metric(ref_name)%} 5 | {% do return(relation) %} 6 | {% else %} 7 | {% do return(api.Relation.create()) %} 8 | {% endif %} 9 | {% endmacro %} -------------------------------------------------------------------------------- /macros/validation/is_valid_dimension.sql: -------------------------------------------------------------------------------- 1 | {% macro is_valid_dimension(dim_name, dimension_list) %} 2 | {% if execute %} 3 | {%- if dim_name not in dimension_list -%} 4 | {%- do exceptions.raise_compiler_error(dim_name ~ " is not a valid dimension") %} 5 | {%- endif -%} 6 | {% endif %} 7 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/metric_on_derived_metric.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | metric('metric_on_derived_metric'), 5 | grain='day', 6 | dimensions=['had_discount','order_country','is_weekend'], 7 | start_date = '2022-01-01', 8 | end_date = '2022-01-05' 9 | 10 | ) 11 | }} -------------------------------------------------------------------------------- /macros/validation/validate_where.sql: -------------------------------------------------------------------------------- 1 | {% macro validate_where(where) %} 2 | 3 | {%- if where is iterable and (where is not string and where is not mapping) -%} 4 | {%- do exceptions.raise_compiler_error("From v0.3.0 onwards, the where clause takes a single string, not a list of filters. Please fix to reflect this change") %} 5 | {%- endif -%} 6 | 7 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/base_count_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | metrics: 3 | - name: base_count_metric 4 | model: ref('fact_orders') 5 | label: Total Discount ($) 6 | timestamp: order_date 7 | time_grains: [day, week, month] 8 | calculation_method: count 9 | expression: order_total 10 | dimensions: 11 | - had_discount 12 | - order_country -------------------------------------------------------------------------------- /integration_tests/models/materialized_models/combined__orders_customers.sql: -------------------------------------------------------------------------------- 1 | with orders as ( 2 | 3 | select * from {{ ref('fact_orders') }} 4 | 5 | ) 6 | , 7 | customers as ( 8 | 9 | select * from {{ ref('dim_customers') }} 10 | 11 | ) 12 | , 13 | final as ( 14 | 15 | select * 16 | from orders 17 | left join customers using (customer_id) 18 | 19 | ) 20 | 21 | select * from final -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/case_when_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | metrics: 3 | - name: case_when_metric 4 | model: ref('fact_orders') 5 | label: Order Total ($) 6 | timestamp: order_date 7 | time_grains: [day, week, month] 8 | calculation_method: sum 9 | expression: case when had_discount = true then 1 else 0 end 10 | dimensions: 11 | - order_country 12 | -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/base_average_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | metrics: 3 | - name: base_average_metric 4 | model: ref('fact_orders') 5 | label: Total Discount ($) 6 | timestamp: order_date 7 | time_grains: [day, week, month, test] 8 | calculation_method: average 9 | expression: discount_total 10 | dimensions: 11 | - had_discount 12 | - order_country -------------------------------------------------------------------------------- /integration_tests/macros/get_nodes_testing.sql: -------------------------------------------------------------------------------- 1 | {% macro get_nodes_testing()%} 2 | 3 | {%- set metric_relation = metric('total_profit') -%} 4 | {{ log("MACRO: Node Unique ID: " ~ metric_relation.unique_id, info=true) }} 5 | {{ log("MACRO: Depends on: " ~ metric_relation.depends_on, info=true) }} 6 | {{ log("MACRO: Depends on nodes: " ~ metric_relation.depends_on.nodes, info=true) }} 7 | 8 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/ratio_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | metrics: 4 | 5 | - name: ratio_metric 6 | label: Ratio ($) 7 | timestamp: order_date 8 | time_grains: [day, week, month] 9 | calculation_method: derived 10 | expression: "{{metric('base_sum_metric')}} / {{metric('base_average_metric')}}" 11 | dimensions: 12 | - had_discount 13 | - order_country -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/base_median_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | metrics: 3 | - name: base_median_metric 4 | model: ref('fact_orders') 5 | label: Total Discount ($) 6 | timestamp: order_date 7 | time_grains: [day, week, month, all_time] 8 | calculation_method: median 9 | expression: discount_total 10 | dimensions: 11 | - had_discount 12 | - order_country 13 | 14 | -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/derived_metric__alternative.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | metrics: 4 | - name: derived_metric__alternative 5 | label: Profit ($) 6 | timestamp: order_date 7 | time_grains: [day, week] 8 | calculation_method: derived 9 | expression: "{{metric('base_sum_metric')}} - {{metric('base_average_metric')}} + 5" 10 | dimensions: 11 | - had_discount 12 | - order_country -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/metric_on_derived_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | metrics: 3 | - name: metric_on_derived_metric 4 | label: Profit Minus Our Civic Duty 5 | timestamp: order_date 6 | time_grains: [day, week] 7 | calculation_method: derived 8 | expression: "{{metric('derived_metric')}} - {{metric('derived_metric__alternative')}}" 9 | dimensions: 10 | - had_discount 11 | - order_country 12 | -------------------------------------------------------------------------------- /integration_tests/seeds/source/dim_customers_source.csv: -------------------------------------------------------------------------------- 1 | customer_id,first_name,last_name,email,gender,is_new_customer,date_added 2 | 1,Geodude,Hills,bhills0@altervista.org,Male,FALSE,2022-01-01 3 | 2,Mew,Coxhead,mcoxhead1@symantec.com,Genderfluid,TRUE,2022-01-06 4 | 3,Mewtwo,Redish,aredish2@last.fm,Genderqueer,FALSE,2022-01-13 5 | 4,Charizard,Basant,lbasant3@dedecms.com,Female,TRUE,2022-02-01 6 | 5,Snorlax,Pokemon,the_email@dedecms.com,Male,TRUE,2022-02-03 -------------------------------------------------------------------------------- /macros/variables/get_relevent_periods.sql: -------------------------------------------------------------------------------- 1 | {%- macro get_relevent_periods(grain, secondary_calculations) %} 2 | 3 | {%- set relevant_periods = [] %} 4 | {%- for calc_config in secondary_calculations if calc_config.period and calc_config.period not in relevant_periods and calc_config.period != grain %} 5 | {%- do relevant_periods.append(calc_config.period) %} 6 | {%- endfor -%} 7 | 8 | {%- do return(relevant_periods)-%} 9 | 10 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/seeds/source/fact_orders_source.csv: -------------------------------------------------------------------------------- 1 | order_id,order_country,order_total,had_discount,customer_id,order_date 2 | 1,Russia,2,false,1,01/28/2022 3 | 2,Mauritius,1,false,2,01/20/2022 4 | 3,Peru,1,false,1,01/13/2022 5 | 4,Kazakhstan,1,true,3,01/06/2022 6 | 5,Portugal,1,false,4,01/08/2022 7 | 6,China,1,false,5,01/21/2022 8 | 7,Germany,1,true,2,01/22/2022 9 | 8,Greenland,1,true,1,02/15/2022 10 | 9,Bangladesh,1,false,2,02/03/2022 11 | 10,Sweden,1,false,3,02/13/2022 -------------------------------------------------------------------------------- /macros/sql_gen/gen_order_by.sql: -------------------------------------------------------------------------------- 1 | {%- macro gen_order_by(grain, dimensions, calendar_dimensions, relevant_periods) -%} 2 | {{ return(adapter.dispatch('gen_order_by', 'metrics')(grain, dimensions, calendar_dimensions, relevant_periods)) }} 3 | {%- endmacro -%} 4 | 5 | {% macro default__gen_order_by(grain, dimensions, calendar_dimensions, relevant_periods) %} 6 | {# #} 7 | {%- if grain %} 8 | order by 1 desc 9 | {% endif -%} 10 | {# #} 11 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/seeds/source/fact_orders_duplicate_source.csv: -------------------------------------------------------------------------------- 1 | order_id,order_country,order_total,had_discount,customer_id,order_date 2 | 1,Russia,2,false,1,01/29/2022 3 | 2,Mauritius,1,false,2,01/21/2022 4 | 3,Peru,1,false,1,01/14/2022 5 | 4,Kazakhstan,1,true,3,01/07/2022 6 | 5,Portugal,1,false,4,01/09/2022 7 | 6,China,1,false,5,01/22/2022 8 | 7,Germany,1,true,2,01/23/2022 9 | 8,Greenland,1,true,1,02/16/2022 10 | 9,Bangladesh,1,false,2,02/04/2022 11 | 10,Sweden,1,false,3,02/14/2022 -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/multiple_metrics__period_to_date.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | [metric('base_sum_metric'), metric('base_average_metric')], 5 | grain='day', 6 | dimensions=['had_discount'], 7 | secondary_calculations=[ 8 | {"calculation": "period_to_date", "aggregate": "min", "period": "year", "alias": "ytd_min"}, 9 | {"calculation": "period_to_date", "aggregate": "max", "period": "month"}, 10 | ] 11 | ) 12 | }} 13 | 14 | -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/multiple_metrics__rolling.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | [metric('base_sum_metric'), metric('base_average_metric')], 5 | grain='day', 6 | dimensions=['had_discount','order_country'], 7 | secondary_calculations=[ 8 | {"calculation": "rolling", "interval": 3, "aggregate": "min", "alias": "min_3mth"}, 9 | {"calculation": "rolling", "interval": 3, "aggregate": "max", "alias": "max_3mth"} 10 | ] 11 | ) 12 | }} -------------------------------------------------------------------------------- /macros/validation/validate_aggregate_coherence.sql: -------------------------------------------------------------------------------- 1 | {% macro validate_aggregate_coherence(metric_aggregate, calculation_aggregate) %} 2 | {% set allowlist = metrics.get_metric_allowlist()[metric_aggregate] %} 3 | 4 | {% if (calculation_aggregate not in allowlist) %} 5 | {% do exceptions.raise_compiler_error("Can't calculate secondary aggregate " ~ calculation_aggregate ~ " when metric's aggregation is " ~ metric_aggregate ~ ". Allowed options are " ~ allowlist ~ ".") %} 6 | {% endif %} 7 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/models/materialized_models/dim_customers.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: dim_customers 4 | columns: 5 | - name: customer_id 6 | description: TBD 7 | 8 | - name: first_name 9 | description: TBD 10 | 11 | - name: last_name 12 | description: TBD 13 | 14 | - name: email 15 | description: TBD 16 | 17 | - name: gender 18 | description: TBD 19 | 20 | - name: is_new_customer 21 | description: TBD 22 | -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/derived_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | metrics: 4 | - name: derived_metric 5 | label: Profit ($) 6 | timestamp: order_date 7 | time_grains: [day, week] 8 | calculation_method: derived 9 | expression: "{{metric('base_sum_metric')}} - {{metric('base_average_metric')}}" 10 | dimensions: 11 | - had_discount 12 | - order_country 13 | 14 | filters: 15 | - field: had_discount 16 | operator: 'is' 17 | value: 'true' 18 | -------------------------------------------------------------------------------- /integration_tests/models/materialized_models/fact_orders.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: fact_orders 4 | columns: 5 | - name: order_id 6 | description: TBD 7 | 8 | - name: order_country 9 | description: TBD 10 | 11 | - name: order_total 12 | description: TBD 13 | 14 | - name: had_discount 15 | description: TBD 16 | 17 | - name: customer_id 18 | description: TBD 19 | 20 | - name: order_date 21 | description: TBD 22 | -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/multiple_metrics__period_over_period.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | [metric('base_sum_metric'), metric('base_average_metric')], 5 | grain='day', 6 | dimensions=['had_discount'], 7 | secondary_calculations=[ 8 | metrics.period_over_period( 9 | comparison_strategy="difference" 10 | ,interval=1 11 | ,metric_list='base_sum_metric' 12 | ) 13 | ] 14 | ) 15 | }} -------------------------------------------------------------------------------- /macros/secondary_calculations/secondary_calculation_prior.sql: -------------------------------------------------------------------------------- 1 | {%- macro default__secondary_calculation_prior(metric_name, grain, dimensions, calc_config, metric_config) -%} 2 | 3 | {%- set calc_sql -%} 4 | lag({{ metric_name }}, {{ calc_config.interval }}) over ( 5 | {% if dimensions -%} 6 | partition by {{ dimensions | join(", ") }} 7 | {% endif -%} 8 | order by date_{{grain}} 9 | ) 10 | {%- endset-%} 11 | {{ calc_sql }} 12 | 13 | {%- endmacro %} 14 | -------------------------------------------------------------------------------- /macros/sql_gen/gen_dimensions_cte.sql: -------------------------------------------------------------------------------- 1 | {%- macro gen_dimensions_cte(group_name, dimensions) -%} 2 | {{ return(adapter.dispatch('gen_dimensions_cte', 'metrics')(group_name, dimensions)) }} 3 | {%- endmacro -%} 4 | 5 | {% macro default__gen_dimensions_cte(group_name, dimensions) %} 6 | 7 | , {{group_name}}__dims as ( 8 | 9 | select distinct 10 | {%- for dim in dimensions %} 11 | {{ dim }}{%- if not loop.last -%},{% endif -%} 12 | {%- endfor %} 13 | from {{group_name}}__aggregate 14 | ) 15 | 16 | {%- endmacro -%} 17 | -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/develop_metric_no_timestamp.sql: -------------------------------------------------------------------------------- 1 | {% set my_metric_yml -%} 2 | {% raw %} 3 | 4 | metrics: 5 | - name: develop_metric 6 | model: ref('fact_orders') 7 | label: Total Discount ($) 8 | calculation_method: average 9 | expression: discount_total 10 | dimensions: 11 | - had_discount 12 | - order_country 13 | 14 | {% endraw %} 15 | {%- endset %} 16 | 17 | select * 18 | from {{ metrics.develop( 19 | develop_yml=my_metric_yml, 20 | metric_list=['develop_metric'], 21 | dimensions=['order_country']) 22 | }} -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Ask the community for help 4 | url: https://github.com/dbt-labs/docs.getdbt.com/discussions 5 | about: Need help troubleshooting? Check out our guide on how to ask 6 | - name: Contact dbt Cloud support 7 | url: mailto:support@getdbt.com 8 | about: Are you using dbt Cloud? Contact our support team for help! 9 | - name: Participate in Discussions 10 | url: https://github.com/dbt-labs/dbt-core/discussions 11 | about: Do you have a Big Idea for dbt? Read open discussions, or start a new one 12 | -------------------------------------------------------------------------------- /models/dbt_metrics_default_calendar.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: dbt_metrics_default_calendar 5 | description: | 6 | An auto generated calendar table that used for metrics. 7 | 8 | columns: 9 | - name: date_day 10 | description: "Date" 11 | - name: date_week 12 | description: "Date truncated into week" 13 | - name: date_month 14 | description: "Date truncated into month" 15 | - name: date_quarter 16 | description: "Date truncated into quarter" 17 | - name: date_year 18 | description: "Date truncated into year" -------------------------------------------------------------------------------- /macros/secondary_calculations/secondary_calculation_period_to_date.sql: -------------------------------------------------------------------------------- 1 | {% macro default__secondary_calculation_period_to_date(metric_name, grain, dimensions, calc_config) %} 2 | {%- set calc_sql -%} 3 | {{- adapter.dispatch('gen_primary_metric_aggregate', 'metrics')(calc_config.aggregate, metric_name) -}} over ( 4 | partition by date_{{ calc_config.period }}{% if dimensions -%}, {{ dimensions | join(", ") }}{%- endif %} 5 | order by date_{{grain}} 6 | rows between unbounded preceding and current row 7 | ) 8 | {%- endset %} 9 | {%- do return (calc_sql) %} 10 | {% endmacro %} -------------------------------------------------------------------------------- /macros/variables/get_non_calendar_dimension_list.sql: -------------------------------------------------------------------------------- 1 | {% macro get_non_calendar_dimension_list(dimensions,calendar_dimensions) %} 2 | 3 | {% set calendar_dims = calendar_dimensions %} 4 | 5 | {# Here we set the calendar as either being the default provided by the package 6 | or the variable provided in the project #} 7 | {% set dimension_list = [] %} 8 | {% for dim in dimensions %} 9 | {%- if dim not in calendar_dimensions -%} 10 | {%- do dimension_list.append(dim | lower) -%} 11 | {%- endif -%} 12 | {% endfor %} 13 | {%- do return(dimension_list) -%} 14 | 15 | {% endmacro %} -------------------------------------------------------------------------------- /.changes/header.tpl.md: -------------------------------------------------------------------------------- 1 | # dbt Core Changelog 2 | 3 | - This file provides a full account of all changes to `dbt-metrics` 4 | - Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. 5 | - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. 6 | - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry) -------------------------------------------------------------------------------- /macros/variables/get_calendar_dimensions.sql: -------------------------------------------------------------------------------- 1 | {% macro get_calendar_dimensions(dimensions) %} 2 | 3 | {% set approved_calendar_dimensions = var('custom_calendar_dimension_list',[]) %} 4 | 5 | {# Here we set the calendar as either being the default provided by the package 6 | or the variable provided in the project #} 7 | {% set calendar_dimensions = [] %} 8 | {% for dim in dimensions %} 9 | {%- if dim in approved_calendar_dimensions -%} 10 | {%- do calendar_dimensions.append(dim | lower) -%} 11 | {%- endif -%} 12 | {% endfor %} 13 | {%- do return(calendar_dimensions) -%} 14 | 15 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | 7 | - dbt run 8 | - dbt test 9 | 10 | ### Resources: 11 | 12 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 13 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 14 | - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support 15 | - Find [dbt events](https://events.getdbt.com) near you 16 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 17 | -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/base_count_distinct_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | metrics: 4 | - name: base_count_distinct_metric 5 | model: ref('fact_orders') 6 | label: Count Distinct 7 | timestamp: order_date 8 | time_grains: [day, week, month] 9 | calculation_method: count_distinct 10 | expression: customer_id 11 | dimensions: 12 | - had_discount 13 | - order_country 14 | window: 15 | count: 14 16 | period: month 17 | filters: 18 | - field: had_discount 19 | operator: '=' 20 | value: 'true' 21 | - field: order_country 22 | operator: '=' 23 | value: "'CA'" -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/simple_develop_metric.sql: -------------------------------------------------------------------------------- 1 | {% set my_metric_yml -%} 2 | 3 | metrics: 4 | - name: develop_metric 5 | model: ref('fact_orders') 6 | label: Total Discount ($) 7 | timestamp: order_date 8 | time_grains: [day, week, month] 9 | calculation_method: average 10 | expression: discount_total 11 | dimensions: 12 | - had_discount 13 | - order_country 14 | config: 15 | treat_null_values_as_zero: false 16 | 17 | {%- endset %} 18 | 19 | select * 20 | from {{ metrics.develop( 21 | develop_yml=my_metric_yml, 22 | metric_list='develop_metric', 23 | grain='month' 24 | ) 25 | }} -------------------------------------------------------------------------------- /models/dbt_metrics_default_calendar.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='table') }} 2 | 3 | with days as ( 4 | {{ metrics.metric_date_spine( 5 | datepart="day", 6 | start_date="cast('1990-01-01' as date)", 7 | end_date="cast('2030-01-01' as date)" 8 | ) 9 | }} 10 | ), 11 | 12 | final as ( 13 | select 14 | cast(date_day as date) as date_day, 15 | cast({{ date_trunc('week', 'date_day') }} as date) as date_week, 16 | cast({{ date_trunc('month', 'date_day') }} as date) as date_month, 17 | cast({{ date_trunc('quarter', 'date_day') }} as date) as date_quarter, 18 | cast({{ date_trunc('year', 'date_day') }} as date) as date_year 19 | from days 20 | ) 21 | 22 | select * from final 23 | -------------------------------------------------------------------------------- /macros/secondary_calculations_configuration/prior.sql: -------------------------------------------------------------------------------- 1 | {% macro prior(interval, alias, metric_list = []) %} 2 | 3 | {% set missing_args = [] %} 4 | {% if not interval %} 5 | {% set _ = missing_args.append("interval") %} 6 | {% endif %} 7 | {% if missing_args | length > 0 %} 8 | {% do exceptions.raise_compiler_error( missing_args | join(", ") ~ ' not provided to prior') %} 9 | {% endif %} 10 | {% if metric_list is string %} 11 | {% set metric_list = [metric_list] %} 12 | {% endif %} 13 | 14 | {% do return ({ 15 | "calculation": "prior", 16 | "interval": interval, 17 | "alias": alias, 18 | "metric_list": metric_list 19 | }) 20 | %} 21 | {% endmacro %} -------------------------------------------------------------------------------- /macros/validation/validate_secondary_calculations.sql: -------------------------------------------------------------------------------- 1 | {% macro validate_secondary_calculations(metric_tree, metrics_dictionary, grain, secondary_calculations) %} 2 | 3 | 4 | {%- for metric_name in metric_tree.base_set %} 5 | {%- for calc_config in secondary_calculations if calc_config.aggregate -%} 6 | {%- do metrics.validate_aggregate_coherence(metric_aggregate=metrics_dictionary[metric_name].calculation_method, calculation_aggregate=calc_config.aggregate) -%} 7 | {%- endfor -%} 8 | {%- endfor -%} 9 | 10 | {%- for calc_config in secondary_calculations if calc_config.period -%} 11 | {%- do metrics.validate_grain_order(metric_grain=grain, calculation_grain=calc_config.period) -%} 12 | {%- endfor -%} 13 | 14 | {% endmacro %} -------------------------------------------------------------------------------- /macros/variables/get_metric_allowlist.sql: -------------------------------------------------------------------------------- 1 | {% macro get_metric_allowlist() %} 2 | {{ return(adapter.dispatch('get_metric_allowlist', 'metrics')()) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_metric_allowlist() %} 6 | {# Keys are the primary aggregation, values are the permitted aggregations to run in secondary calculations. #} 7 | {% do return ({ 8 | "average": ['min', 'max'], 9 | "median": ['min', 'max'], 10 | "count": ['min', 'max', 'sum', 'average'], 11 | "count_distinct": ['min', 'max', 'sum', 'average'], 12 | "sum": ['min', 'max', 'sum', 'average'], 13 | "max": ['min', 'max', 'sum', 'average'], 14 | "min": ['min', 'max', 'sum', 'average'], 15 | "derived": ['min', 'max', 'sum'], 16 | }) %} 17 | {% endmacro %} -------------------------------------------------------------------------------- /macros/secondary_calculations_configuration/rolling.sql: -------------------------------------------------------------------------------- 1 | {% macro rolling(aggregate, interval, alias, metric_list=[]) %} 2 | 3 | {% set missing_args = [] %} 4 | {% if not aggregate %} 5 | {% set _ = missing_args.append("aggregate") %} 6 | {% endif %} 7 | {% if missing_args | length > 0 %} 8 | {% do exceptions.raise_compiler_error( missing_args | join(", ") ~ ' not provided to rolling') %} 9 | {% endif %} 10 | {% if metric_list is string %} 11 | {% set metric_list = [metric_list] %} 12 | {% endif %} 13 | 14 | {% do return ({ 15 | "calculation": "rolling", 16 | "aggregate": aggregate, 17 | "interval": interval, 18 | "alias": alias, 19 | "metric_list": metric_list 20 | }) 21 | %} 22 | {% endmacro %} -------------------------------------------------------------------------------- /.github/workflows/jira-creation.yml: -------------------------------------------------------------------------------- 1 | # **what?** 2 | # Mirrors issues into Jira. Includes the information: title, 3 | # GitHub Issue ID and URL 4 | 5 | # **why?** 6 | # Jira is our tool for tracking and we need to see these issues in there 7 | 8 | # **when?** 9 | # On issue creation or when an issue is labeled `Jira` 10 | 11 | name: Jira Issue Creation 12 | 13 | on: 14 | issues: 15 | types: [opened, labeled] 16 | 17 | permissions: 18 | issues: write 19 | 20 | jobs: 21 | call-creation-action: 22 | uses: dbt-labs/actions/.github/workflows/jira-creation.yml@main 23 | with: 24 | project_key: "SEMANTIC" 25 | secrets: 26 | JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} 27 | JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} 28 | JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} -------------------------------------------------------------------------------- /macros/secondary_calculations/secondary_calculation_rolling.sql: -------------------------------------------------------------------------------- 1 | {%- macro default__secondary_calculation_rolling(metric_name, grain, dimensions, calc_config) %} 2 | {%- set calc_sql -%} 3 | {{ adapter.dispatch('gen_primary_metric_aggregate', 'metrics')(calc_config.aggregate, metric_name) }} over ( 4 | {% if dimensions -%} 5 | partition by {{ dimensions | join(", ") }} 6 | {%- endif %} 7 | order by date_{{grain}} 8 | {%- if calc_config.interval %} 9 | rows between {{ calc_config.interval - 1 }} preceding and current row 10 | {%- else %} 11 | rows between unbounded preceding and current row 12 | {%- endif %} 13 | ) 14 | {%- endset %} 15 | {% do return (calc_sql) %} 16 | {%- endmacro %} -------------------------------------------------------------------------------- /.github/workflows/jira-label.yml: -------------------------------------------------------------------------------- 1 | # **what?** 2 | # Calls mirroring Jira label Action. Includes adding a new label 3 | # to an existing issue or removing a label as well 4 | 5 | # **why?** 6 | # Jira is our tool for tracking and we need to see these labels in there 7 | 8 | # **when?** 9 | # On labels being added or removed from issues 10 | 11 | name: Jira Label Mirroring 12 | 13 | on: 14 | issues: 15 | types: [labeled, unlabeled] 16 | 17 | permissions: 18 | issues: read 19 | 20 | jobs: 21 | call-label-action: 22 | uses: dbt-labs/actions/.github/workflows/jira-label.yml@main 23 | with: 24 | project_key: "SEMANTIC" 25 | secrets: 26 | JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} 27 | JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} 28 | JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} 29 | -------------------------------------------------------------------------------- /examples/metric_jsonschema_example.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "dbt_metric_file", 3 | "type": "object", 4 | "required": ["name","label","timestamp","time_grains","calculation_method","expression"], 5 | "additionalProperties": false, 6 | "properties": { 7 | "name": { "type": "string"}, 8 | "label": { "type": "string"}, 9 | "timestamp": { "type": "string"}, 10 | "time_grains": { 11 | "type": "array", 12 | "items":{ 13 | "type":"string" 14 | } 15 | }, 16 | "calculation_method": { "type": "string"}, 17 | "expression": { "type": "string"}, 18 | "dimensions": { 19 | "type": "array", 20 | "items":{ 21 | "type":"string" 22 | } 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /macros/variables/get_total_dimension_count.sql: -------------------------------------------------------------------------------- 1 | {%- macro get_total_dimension_count(grain, dimensions, calendar_dimensions, relevant_periods) %} 2 | 3 | {# This macro calcualtes the total amount of dimensions that will need to be grouped by #} 4 | 5 | {%- set dimension_length = dimensions | length -%} 6 | {%- set calendar_dimension_length = calendar_dimensions | length -%} 7 | 8 | {%- if grain -%} 9 | {%- set grain_length = 1 -%} 10 | {%- else -%} 11 | {%- set grain_length = 0 -%} 12 | {%- endif -%} 13 | 14 | {%- set cleaned_relevant_periods = [] -%} 15 | {%- set period_length = relevant_periods | length -%} 16 | {%- set total_length = grain_length + dimension_length + period_length + calendar_dimension_length -%} 17 | 18 | {% do return(total_length) %} 19 | 20 | {% endmacro %} -------------------------------------------------------------------------------- /.github/workflows/jira-transition.yml: -------------------------------------------------------------------------------- 1 | # **what?** 2 | # Transition a Jira issue to a new state 3 | # Only supports these GitHub Issue transitions: 4 | # closed, deleted, reopened 5 | 6 | # **why?** 7 | # Jira needs to be kept up-to-date 8 | 9 | # **when?** 10 | # On issue closing, deletion, reopened 11 | 12 | name: Jira Issue Transition 13 | 14 | on: 15 | issues: 16 | types: [closed, deleted, reopened] 17 | 18 | # no special access is needed 19 | permissions: read-all 20 | 21 | jobs: 22 | call-transition-action: 23 | uses: dbt-labs/actions/.github/workflows/jira-transition.yml@main 24 | with: 25 | project_key: "SEMANTIC" 26 | secrets: 27 | JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} 28 | JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} 29 | JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} 30 | -------------------------------------------------------------------------------- /macros/validation/validate_timestamp.sql: -------------------------------------------------------------------------------- 1 | {% macro validate_timestamp(grain, metric_tree, metrics_dictionary, dimensions) %} 2 | 3 | {# We check the metrics being used and if there is no grain we ensure that 4 | none of the dimensions provided are from the calendar #} 5 | {% if not grain %} 6 | {%- if metrics.get_calendar_dimensions(dimensions) | length > 0 -%} 7 | 8 | {% for metric_name in metric_tree.full_set %} 9 | {% set metric_relation = metrics_dictionary[metric_name]%} 10 | {% if not metric_relation.timestamp %} 11 | {%- do exceptions.raise_compiler_error("The metric " ~ metric_name ~ " is using a calendar dimension but does not have a timestamp defined.") %} 12 | {% endif %} 13 | {% endfor %} 14 | 15 | {% endif %} 16 | {% endif %} 17 | 18 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/base_count_metric__secondary_calculations.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from 3 | {{ metrics.calculate( 4 | metric('base_count_metric'), 5 | grain='month', 6 | start_date = '2022-01-01', 7 | end_date = '2022-04-01', 8 | secondary_calculations=[ 9 | {"calculation": "period_over_period", "interval": 1, "comparison_strategy": "difference", "alias": "pop_1mth"}, 10 | {"calculation": "period_over_period", "interval": 1, "comparison_strategy": "ratio"}, 11 | {"calculation": "period_to_date", "aggregate": "sum", "period": "year", "alias": "ytd_sum"}, 12 | {"calculation": "period_to_date", "aggregate": "max", "period": "month"}, 13 | {"calculation": "rolling", "interval": 3, "aggregate": "average", "alias": "avg_3mth"}, 14 | {"calculation": "rolling", "aggregate": "sum"}, 15 | ] 16 | ) 17 | }} -------------------------------------------------------------------------------- /.github/actions/end-to-end-test/action.yml: -------------------------------------------------------------------------------- 1 | name: "End to end testing" 2 | description: "Set up profile and run dbt with test project" 3 | inputs: 4 | dbt_target: 5 | description: "Name of target to use when running dbt" 6 | required: true 7 | 8 | 9 | runs: 10 | using: "composite" 11 | steps: 12 | 13 | - name: Install python dependencies 14 | shell: bash 15 | run: | 16 | pip install --user --upgrade pip 17 | pip --version 18 | pip install -r dev-requirements.txt 19 | 20 | ## Make sure to defined dbt_target as an environment variable 21 | ## Previously we were supplying just as an input and os.environ 22 | ## wasn't able to recognize it. 23 | - name: Run pytest 24 | shell: bash 25 | env: 26 | dbt_target: ${{ inputs.dbt_target }} 27 | run: | 28 | python3 -m pytest tests/functional -------------------------------------------------------------------------------- /macros/variables/get_metrics_dictionary.sql: -------------------------------------------------------------------------------- 1 | {% macro get_metrics_dictionary(metric_tree, develop_yml = none) %} 2 | 3 | {% set metrics_dictionary = {} %} 4 | 5 | {% for metric_name in metric_tree.full_set %} 6 | {% if develop_yml is not none %} 7 | {% set metric_object = develop_yml[metric_name]%} 8 | {% else %} 9 | {% set metric_object = metrics.get_metric_relation(metric_name) %} 10 | {% endif %} 11 | {% set metric_definition = metrics.get_metric_definition(metric_object) %} 12 | {% if not metric_definition.config %} 13 | {% do metric_definition.update({'config':{}}) %} 14 | {% endif %} 15 | {% do metrics_dictionary.update({metric_name:{}})%} 16 | {% do metrics_dictionary.update({metric_name:metric_definition})%} 17 | {% endfor %} 18 | 19 | {% do return(metrics_dictionary) %} 20 | 21 | {% endmacro %} -------------------------------------------------------------------------------- /macros/validation/validate_grain_order.sql: -------------------------------------------------------------------------------- 1 | {% macro validate_grain_order(metric_grain, calculation_grain) %} 2 | {% set grains = metrics.get_grain_order() %} 3 | 4 | {% if metric_grain not in grains or calculation_grain not in grains %} 5 | {% set comma = joiner(", ") %} 6 | {% do exceptions.raise_compiler_error("Unknown grains: [" ~ (comma() ~ metric_grain if metric_grain not in grains) ~ (comma() ~ calculation_grain if calculation_grain not in grains) ~ "]") %} 7 | {% endif %} 8 | 9 | {% set metric_grain_index = grains.index(metric_grain) %} 10 | {% set calculation_grain_index = grains.index(calculation_grain) %} 11 | 12 | {% if (calculation_grain_index < metric_grain_index) %} 13 | {% do exceptions.raise_compiler_error("Can't calculate secondary metric at " ~ calculation_grain ~"-level when metric is at " ~ metric_grain ~ "-level") %} 14 | {% endif %} 15 | {% endmacro %} -------------------------------------------------------------------------------- /macros/secondary_calculations_configuration/period_to_date.sql: -------------------------------------------------------------------------------- 1 | {% macro period_to_date(aggregate, period, alias, metric_list = []) %} 2 | 3 | {% set missing_args = [] %} 4 | {% if not aggregate %} 5 | {% set _ = missing_args.append("aggregate") %} 6 | {% endif %} 7 | {% if not period %} 8 | {% set _ = missing_args.append("period") %} 9 | {% endif %} 10 | {% if missing_args | length > 0 %} 11 | {% do exceptions.raise_compiler_error( missing_args | join(", ") ~ ' not provided to period_to_date') %} 12 | {% endif %} 13 | {% if metric_list is string %} 14 | {% set metric_list = [metric_list] %} 15 | {% endif %} 16 | 17 | {% do return ({ 18 | "calculation": "period_to_date", 19 | "aggregate": aggregate, 20 | "period": period, 21 | "alias": alias, 22 | "metric_list": metric_list 23 | }) 24 | %} 25 | {% endmacro %} -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-dotenv 3 | 4 | # Bleeding edge 5 | # git+https://github.com/dbt-labs/dbt-core.git@main#egg=dbt-tests-adapter&subdirectory=tests/adapter 6 | # git+https://github.com/dbt-labs/dbt-core.git@main#egg=dbt-core&subdirectory=core 7 | # git+https://github.com/dbt-labs/dbt-core.git@main#egg=dbt-postgres&subdirectory=plugins/postgres 8 | # git+https://github.com/dbt-labs/dbt-redshift.git 9 | # git+https://github.com/dbt-labs/dbt-snowflake.git 10 | # git+https://github.com/dbt-labs/dbt-bigquery.git 11 | # git+https://github.com/databricks/dbt-databricks.git 12 | 13 | # Most recent stable release 14 | # dbt-tests-adapter 15 | # dbt-core 16 | # dbt-redshift 17 | # dbt-snowflake 18 | # dbt-bigquery 19 | # dbt-databricks 20 | 21 | # Most recent release candidates 22 | dbt-tests-adapter==1.5.0-rc1 23 | dbt-core==1.5.0-rc1 24 | dbt-redshift==1.5.0-rc1 25 | dbt-snowflake==1.5.0-rc1 26 | dbt-bigquery==1.5.0-rc1 27 | # dbt-databricks==1.5.0-rc1 28 | -------------------------------------------------------------------------------- /integration_tests/seeds/expected/base_count_metric__secondary_calculations_expected.csv: -------------------------------------------------------------------------------- 1 | date_month,date_year,has_messaged,is_active_past_quarter,base_count_metric,base_count_metric_pop_1mth,base_count_metric_ratio_to_1_month_ago,base_count_metric_ytd_sum,base_count_metric_max_for_month,base_count_metric_avg_3mth,base_count_metric_rolling_sum_3_month 2 | 2021-01-01,2021-01-01,TRUE,FALSE,0,0,,0,0,0.000,0 3 | 2021-02-01,2021-01-01,TRUE,FALSE,1,1,,1,1,0.500,1 4 | 2021-03-01,2021-01-01,TRUE,FALSE,0,-1,0,1,0,0.333,1 5 | 2021-04-01,2021-01-01,TRUE,FALSE,0,0,,1,0,0.333,1 6 | 2021-01-01,2021-01-01,TRUE,TRUE,1,1,,1,1,1.000,1 7 | 2021-02-01,2021-01-01,TRUE,TRUE,0,-1,0,1,0,0.500,1 8 | 2021-03-01,2021-01-01,TRUE,TRUE,0,0,,1,0,0.333,1 9 | 2021-04-01,2021-01-01,TRUE,TRUE,0,0,,1,0,0.000,0 10 | 2021-01-01,2021-01-01,FALSE,FALSE,0,0,,0,0,0.000,0 11 | 2021-02-01,2021-01-01,FALSE,FALSE,0,0,,0,0,0.000,0 12 | 2021-03-01,2021-01-01,FALSE,FALSE,0,0,,0,0,0.000,0 13 | 2021-04-01,2021-01-01,FALSE,FALSE,1,1,,1,1,0.333,1 -------------------------------------------------------------------------------- /macros/secondary_calculations_configuration/period_over_period.sql: -------------------------------------------------------------------------------- 1 | {% macro period_over_period(comparison_strategy, interval, alias, metric_list = []) %} 2 | 3 | {% set missing_args = [] %} 4 | {% if not comparison_strategy %} 5 | {% set _ = missing_args.append("comparison_strategy") %} 6 | {% endif %} 7 | {% if not interval %} 8 | {% set _ = missing_args.append("interval") %} 9 | {% endif %} 10 | {% if missing_args | length > 0 %} 11 | {% do exceptions.raise_compiler_error( missing_args | join(", ") ~ ' not provided to period_over_period') %} 12 | {% endif %} 13 | {% if metric_list is string %} 14 | {% set metric_list = [metric_list] %} 15 | {% endif %} 16 | 17 | {% do return ({ 18 | "calculation": "period_over_period", 19 | "comparison_strategy": comparison_strategy, 20 | "interval": interval, 21 | "alias": alias, 22 | "metric_list": metric_list 23 | }) 24 | %} 25 | {% endmacro %} -------------------------------------------------------------------------------- /macros/variables/get_develop_unique_metric_id_list.sql: -------------------------------------------------------------------------------- 1 | {%- macro get_develop_unique_metric_id_list(metric_definition) %} 2 | 3 | {% set re = modules.re %} 4 | 5 | {%- set metric_list = [] -%} 6 | 7 | {%- if metric_definition.calculation_method == 'derived' %} 8 | 9 | {# First we get the list of nodes that this metric is dependent on. This is inclusive 10 | of all parent metrics and SHOULD only contain parent metrics #} 11 | {%- set dependency_metrics = re.findall("'[^']+'",metric_definition.expression) -%} 12 | 13 | {# This part is suboptimal - we're looping through the dependent nodes and extracting 14 | the model name from the idenitfier. Ideally we'd just use the metrics attribute but 15 | right now its a list of lists #} 16 | {%- for metric_name in dependency_metrics -%} 17 | = {% do metric_list.append(metric_name.replace('\'','')) %} 18 | {%- endfor -%} 19 | 20 | {%- endif %} 21 | 22 | {% do return(metric_list) %} 23 | 24 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/dbt_project.yml: -------------------------------------------------------------------------------- 1 | # Name your project! Project names should contain only lowercase characters 2 | # and underscores. A good package name should reflect your organization's 3 | # name or the intended use of these models 4 | name: "dbt_metrics_integration_tests" 5 | version: "1.0.0" 6 | config-version: 2 7 | 8 | # This setting configures which "profile" dbt uses for this project. 9 | profile: "dbt_metrics_integration_tests" 10 | 11 | model-paths: ["models"] 12 | analysis-paths: ["analyses"] 13 | test-paths: ["tests"] 14 | seed-paths: ["seeds"] 15 | macro-paths: ["macros"] 16 | snapshot-paths: ["snapshots"] 17 | 18 | target-path: "target" 19 | clean-targets: 20 | - "target" 21 | - "dbt_packages" 22 | - "logs" 23 | 24 | models: 25 | 26 | dbt_metrics_integration_tests: 27 | 28 | metric_testing_models: 29 | +materialized: table 30 | 31 | materialized_models: 32 | +materialized: table 33 | 34 | vars: 35 | dbt_metrics_calendar_model: custom_calendar 36 | custom_calendar_dimension_list: ["is_weekend"] 37 | -------------------------------------------------------------------------------- /macros/sql_gen/gen_calendar_cte.sql: -------------------------------------------------------------------------------- 1 | {%- macro gen_calendar_cte(calendar_tbl, start_date, end_date) -%} 2 | {{ return(adapter.dispatch('gen_calendar_cte', 'metrics')(calendar_tbl, start_date, end_date)) }} 3 | {%- endmacro -%} 4 | 5 | {%- macro default__gen_calendar_cte(calendar_tbl, start_date, end_date) %} 6 | 7 | with calendar as ( 8 | {# This CTE creates our base calendar and then limits the date range for the 9 | start and end date provided by the macro call -#} 10 | select 11 | * 12 | from {{ calendar_tbl }} 13 | {% if start_date or end_date %} 14 | {%- if start_date and end_date -%} 15 | where date_day >= cast('{{ start_date }}' as date) 16 | and date_day <= cast('{{ end_date }}' as date) 17 | {%- elif start_date and not end_date -%} 18 | where date_day >= cast('{{ start_date }}' as date) 19 | {%- elif end_date and not start_date -%} 20 | where date_day <= cast('{{ end_date }}' as date) 21 | {%- endif -%} 22 | {% endif %} 23 | ) 24 | 25 | {%- endmacro -%} 26 | -------------------------------------------------------------------------------- /macros/graph_parsing/get_model_relation.sql: -------------------------------------------------------------------------------- 1 | {% macro get_model_relation(ref_name, metric_name=None) %} 2 | 3 | {% if execute %} 4 | {% set model_ref_node = graph.nodes.values() | selectattr('name', 'equalto', ref_name) | first %} 5 | {% if model_ref_node | length == 0 %} 6 | {%- do exceptions.raise_compiler_error("The metric " ~ metric_name ~ " is referencing the model " ~ ref_name ~ ", which does not exist.") %} 7 | {% endif %} 8 | 9 | {% set relation = api.Relation.create( 10 | database = model_ref_node.database, 11 | schema = model_ref_node.schema, 12 | identifier = model_ref_node.alias 13 | ) 14 | %} 15 | 16 | {% if model_ref_node.config.materialized == "ephemeral" %} 17 | {%- do exceptions.raise_compiler_error("The resource " ~ relation.name ~ " is an ephemeral model which is not supported") %} 18 | {% endif%} 19 | 20 | {% do return(relation) %} 21 | 22 | {% else %} 23 | {% do return(api.Relation.create()) %} 24 | {% endif %} 25 | 26 | {% endmacro %} -------------------------------------------------------------------------------- /.github/workflows/triage-labels.yml: -------------------------------------------------------------------------------- 1 | # **what?** 2 | # When the core team triages, we sometimes need more information from the issue creator. In 3 | # those cases we remove the `triage` label and add the `awaiting_response` label. Once we 4 | # recieve a response in the form of a comment, we want the `awaiting_response` label removed 5 | # in favor of the `triage` label so we are aware that the issue needs action. 6 | 7 | # **why?** 8 | # To help with out team triage issue tracking 9 | 10 | # **when?** 11 | # This will run when a comment is added to an issue and that issue has to `awaiting_response` label. 12 | 13 | name: Update Triage Label 14 | 15 | on: issue_comment 16 | 17 | defaults: 18 | run: 19 | shell: bash 20 | 21 | permissions: 22 | issues: write 23 | 24 | jobs: 25 | triage_label: 26 | if: contains(github.event.issue.labels.*.name, 'awaiting_response') 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: initial labeling 30 | uses: andymckay/labeler@3a4296e9dcdf9576b0456050db78cfd34853f260 # andymckay/labeler@master 31 | with: 32 | add-labels: "triage" 33 | remove-labels: "awaiting_response" -------------------------------------------------------------------------------- /macros/validation/validate_derived_metrics.sql: -------------------------------------------------------------------------------- 1 | {% macro validate_derived_metrics(metric_tree) %} 2 | 3 | {# We loop through the full set here to ensure that metrics that aren't listed 4 | as derived are not dependent on another metric. #} 5 | 6 | {% for metric_name in metric_tree.full_set %} 7 | {% set metric_relation = metric(metric_name)%} 8 | {% if metric_relation.calculation_method == "derived" and metric_relation.filters | length > 0 %} 9 | {%- do exceptions.raise_compiler_error("Derived metrics, such as " ~ metric_relation.name ~", do not support the use of filters. ") %} 10 | {%- endif %} 11 | {% set metric_relation_depends_on = metric_relation.metrics | join (",") %} 12 | {% if metric_relation.calculation_method != "derived" and metric_relation.metrics | length > 0 %} 13 | {%- do exceptions.raise_compiler_error("The metric " ~ metric_relation.name ~" also references '" ~ metric_relation_depends_on ~ "' but its calculation method is '" ~ metric_relation.calculation_method ~ "'. Only metrics of calculation method derived can reference other metrics.") %} 14 | {%- endif %} 15 | {% endfor %} 16 | 17 | {% endmacro %} -------------------------------------------------------------------------------- /macros/sql_gen/gen_secondary_calculations.sql: -------------------------------------------------------------------------------- 1 | {%- macro gen_secondary_calculations(metric_tree, metrics_dictionary, grain, dimensions, secondary_calculations, calendar_dimensions) -%} 2 | {{ return(adapter.dispatch('gen_secondary_calculations', 'metrics')(metric_tree, metrics_dictionary, grain, dimensions, secondary_calculations, calendar_dimensions)) }} 3 | {%- endmacro -%} 4 | 5 | {% macro default__gen_secondary_calculations(metric_tree, metrics_dictionary, grain, dimensions, secondary_calculations, calendar_dimensions) %} 6 | 7 | {%- for calc_config in secondary_calculations %} 8 | {%- if calc_config.metric_list | length > 0 -%} 9 | {%- for metric_name in calc_config.metric_list -%} 10 | ,{{ metrics.perform_secondary_calculation(metric_name, grain, dimensions, calendar_dimensions, calc_config, metrics_dictionary[metric_name].config) }} 11 | {%- endfor %} 12 | {%- else %} 13 | {%- for metric_name in metric_tree.base_set -%} 14 | , {{ metrics.perform_secondary_calculation(metric_name, grain, dimensions, calendar_dimensions, calc_config, metrics_dictionary[metric_name].config) }} 15 | {%- endfor %} 16 | {%- endif %} 17 | {%- endfor %} 18 | 19 | {%- endmacro %} 20 | -------------------------------------------------------------------------------- /dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | # Name your project! Project names should contain only lowercase characters 3 | # and underscores. A good package name should reflect your organization's 4 | # name or the intended use of these models 5 | name: 'metrics' 6 | version: '1.0.0' 7 | config-version: 2 8 | 9 | # This setting configures which "profile" dbt uses for this project. 10 | profile: 'dbt_metrics_integration_tests' 11 | 12 | # These configurations specify where dbt should look for different types of files. 13 | # The `source-paths` config, for example, states that models in this project can be 14 | # found in the "models/" directory. You probably won't need to change these! 15 | model-paths: ["models"] 16 | analysis-paths: ["analyses"] 17 | test-paths: ["tests"] 18 | seed-paths: ["seeds"] 19 | macro-paths: ["macros"] 20 | snapshot-paths: ["snapshots"] 21 | 22 | require-dbt-version: [">=1.5.0-a1", "<1.6.0"] 23 | # require-dbt-version: [">=1.4.0-a1", "<1.5.0"] 24 | # require-dbt-version: [">=1.3.0-a1", "<1.4.0"] 25 | 26 | target-path: "target" # directory which will store compiled SQL files 27 | clean-targets: # directories to be removed by `dbt clean` 28 | - "target" 29 | - "dbt_packages" 30 | 31 | models: 32 | +materialized: ephemeral -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: "Close stale issues and PRs" 2 | on: 3 | schedule: 4 | - cron: "30 1 * * *" 5 | 6 | permissions: 7 | issues: write 8 | pull-requests: write 9 | 10 | jobs: 11 | stale: 12 | runs-on: ubuntu-latest 13 | steps: 14 | # pinned at v4 (https://github.com/actions/stale/releases/tag/v4.0.0) 15 | - uses: actions/stale@cdf15f641adb27a71842045a94023bef6945e3aa 16 | with: 17 | stale-issue-message: "This issue has been marked as Stale because it has been open for 180 days with no activity. If you would like the issue to remain open, please remove the stale label or comment on the issue, or it will be closed in 7 days." 18 | stale-pr-message: "This PR has been marked as Stale because it has been open for 180 days with no activity. If you would like the PR to remain open, please remove the stale label or comment on the PR, or it will be closed in 7 days." 19 | close-issue-message: "Although we are closing this issue as stale, it's not gone forever. Issues can be reopened if there is renewed community interest; add a comment to notify the maintainers." 20 | # mark issues/PRs stale when they haven't seen activity in 180 days 21 | days-before-stale: 180 -------------------------------------------------------------------------------- /integration_tests/models/metric_testing_models/develop_metric.sql: -------------------------------------------------------------------------------- 1 | {% set my_metric_yml -%} 2 | {% raw %} 3 | 4 | metrics: 5 | - name: develop_metric 6 | model: ref('fact_orders') 7 | label: Total Discount ($) 8 | timestamp: order_date 9 | time_grains: [day, week, month] 10 | calculation_method: average 11 | expression: discount_total 12 | dimensions: 13 | - had_discount 14 | - order_country 15 | 16 | - name: derived_metric 17 | label: Total Discount ($) 18 | timestamp: order_date 19 | time_grains: [day, week, month] 20 | calculation_method: derived 21 | expression: "{{ metric('develop_metric') }} - 1 " 22 | dimensions: 23 | - had_discount 24 | - order_country 25 | 26 | - name: some_other_metric_not_using 27 | label: Total Discount ($) 28 | timestamp: order_date 29 | time_grains: [day, week, month] 30 | calculation_method: derived 31 | expression: "{{ metric('derived_metric') }} - 1 " 32 | dimensions: 33 | - had_discount 34 | - order_country 35 | 36 | {% endraw %} 37 | {%- endset %} 38 | 39 | select * 40 | from {{ metrics.develop( 41 | develop_yml=my_metric_yml, 42 | metric_list=['derived_metric'], 43 | grain='month' 44 | ) 45 | }} -------------------------------------------------------------------------------- /macros/variables/get_metric_list.sql: -------------------------------------------------------------------------------- 1 | {%- macro get_metric_list(metric) %} 2 | 3 | {%- if metric.metrics | length > 0 %} 4 | 5 | {# First we get the list of nodes that this metric is dependent on. This is inclusive 6 | of all parent metrics and SHOULD only contain parent metrics #} 7 | {%- set node_list = metric.depends_on.nodes -%} 8 | {%- set metric_list = [] -%} 9 | 10 | {# This part is suboptimal - we're looping through the dependent nodes and extracting 11 | the model name from the idenitfier. Ideally we'd just use the metrics attribute but 12 | right now its a list of lists #} 13 | {%- for node in node_list -%} 14 | {% set metric_name = node.split('.')[2] %} 15 | {% do metric_list.append(metric_name) %} 16 | {%- endfor -%} 17 | 18 | {% else %} 19 | 20 | {# For non-derived metrics, we just need the relation of the base model ie 21 | the model that its built. Then we append it to the metric list name so the same 22 | variable used in derived metrics can be used below #} 23 | {%- set metric_list = [] -%} 24 | {% do metric_list.append(metric.name) %} 25 | 26 | {%- endif %} 27 | 28 | {% do return(metric_list) %} 29 | 30 | {% endmacro %} -------------------------------------------------------------------------------- /macros/variables/get_metric_unique_id_list.sql: -------------------------------------------------------------------------------- 1 | {%- macro get_metric_unique_id_list(metric) %} 2 | 3 | {%- if metric.metrics | length > 0 %} 4 | 5 | {# First we get the list of nodes that this metric is dependent on. This is inclusive 6 | of all parent metrics and SHOULD only contain parent metrics #} 7 | {%- set node_list = metric.depends_on.nodes -%} 8 | {%- set metric_list = [] -%} 9 | 10 | {# This part is suboptimal - we're looping through the dependent nodes and extracting 11 | the model name from the idenitfier. Ideally we'd just use the metrics attribute but 12 | right now its a list of lists #} 13 | {%- for node in node_list -%} 14 | {%- if node.split('.')[0] == 'metric' -%} 15 | {% do metric_list.append(node.split('.')[2]) %} 16 | {%- endif -%} 17 | {%- endfor -%} 18 | 19 | {% else %} 20 | 21 | {# For non-derived metrics, we just need the relation of the base model ie 22 | the model that its built. Then we append it to the metric list name so the same 23 | variable used in derived metrics can be used below #} 24 | {%- set metric_list = [] -%} 25 | 26 | {%- endif %} 27 | 28 | {% do return(metric_list) %} 29 | 30 | {% endmacro %} -------------------------------------------------------------------------------- /test.env.example: -------------------------------------------------------------------------------- 1 | #-------------------------------------------------------- 2 | #Target (change this to target tests need to run against) 3 | #-------------------------------------------------------- 4 | dbt_target=postgres 5 | 6 | #-------------------------------------------------------- 7 | #PostgreSQL (dbt_target: postgres) 8 | #-------------------------------------------------------- 9 | POSTGRES_TEST_HOST=localhost 10 | POSTGRES_TEST_USER=root 11 | POSTGRES_TEST_PASSWORD=password 12 | POSTGRES_TEST_PORT=5432 13 | POSTGRES_TEST_DB=dbt 14 | 15 | #-------------------------------------------------------- 16 | #Redshift (dbt_target: redshift) 17 | #-------------------------------------------------------- 18 | REDSHIFT_TEST_HOST= 19 | REDSHIFT_TEST_USER= 20 | REDSHIFT_TEST_PASS= 21 | REDSHIFT_TEST_DBNAME= 22 | REDSHIFT_TEST_PORT= 23 | 24 | #-------------------------------------------------------- 25 | #Snowflake (dbt_target: snowflake) 26 | #-------------------------------------------------------- 27 | SNOWFLAKE_TEST_ACCOUNT= 28 | SNOWFLAKE_TEST_USER= 29 | SNOWFLAKE_TEST_PASSWORD= 30 | SNOWFLAKE_TEST_ROLE= 31 | SNOWFLAKE_TEST_DATABASE= 32 | SNOWFLAKE_TEST_WAREHOUSE= 33 | 34 | #-------------------------------------------------------- 35 | #BigQuery (dbt_target: bigquery) 36 | #-------------------------------------------------------- 37 | BIGQUERY_TEST_PROJECT= 38 | BIGQUERY_SERVICE_KEY_PATH= -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | ## What is this PR? 3 | This is a: 4 | - [ ] documentation update 5 | - [ ] bug fix with no breaking changes 6 | - [ ] new functionality 7 | - [ ] a breaking change 8 | 9 | All pull requests from community contributors should target the `main` branch (default). 10 | 11 | ## Description & motivation 12 | 15 | 16 | ## Checklist 17 | - [ ] I have verified that these changes work locally on the following warehouses (Note: it's okay if you do not have access to all warehouses, this helps us understand what has been covered) 18 | - [ ] BigQuery 19 | - [ ] Postgres 20 | - [ ] Redshift 21 | - [ ] Snowflake 22 | - [ ] Databricks 23 | - [ ] I have updated the README.md (if applicable) 24 | - [ ] I have added tests & descriptions to my models (and macros if applicable) 25 | - [ ] I have added an entry to CHANGELOG.md 26 | 27 | --- 28 | ### Tenets to keep in mind 29 | - A metric value should be consistent everywhere that it is referenced 30 | - We prefer generalized metrics with many dimensions over specific metrics with few dimensions 31 | - It should be easier to use dbt’s metrics than it is to avoid them 32 | - Organization and discoverability are as important as precision 33 | - One-off models built to power metrics are an anti-pattern 34 | -------------------------------------------------------------------------------- /macros/sql_gen/gen_spine_time_cte.sql: -------------------------------------------------------------------------------- 1 | {%- macro gen_spine_time_cte(group_name, grain, dimensions, secondary_calculations, relevant_periods, calendar_dimensions, dimensions_provided) -%} 2 | {{ return(adapter.dispatch('gen_spine_time_cte', 'metrics')(group_name, grain, dimensions, secondary_calculations, relevant_periods, calendar_dimensions, dimensions_provided)) }} 3 | {%- endmacro -%} 4 | 5 | {% macro default__gen_spine_time_cte(group_name, grain, dimensions, secondary_calculations, relevant_periods, calendar_dimensions, dimensions_provided) %} 6 | 7 | , {{group_name}}__spine_time as ( 8 | 9 | select 10 | calendar.date_{{grain}} 11 | {%- if secondary_calculations | length > 0 -%} 12 | {% for period in relevant_periods %} 13 | {%- if period != grain -%} 14 | , calendar.date_{{ period }} 15 | {%- endif -%} 16 | {% endfor -%} 17 | {% endif -%} 18 | {% for calendar_dim in calendar_dimensions %} 19 | , calendar.{{ calendar_dim }} 20 | {%- endfor %} 21 | {%- for dim in dimensions %} 22 | , {{group_name}}__dims.{{ dim }} 23 | {%- endfor %} 24 | from calendar 25 | {%- if dimensions_provided %} 26 | cross join {{group_name}}__dims 27 | {%- endif %} 28 | {{ metrics.gen_group_by(grain,dimensions,calendar_dimensions,relevant_periods) }} 29 | 30 | ) 31 | {%- endmacro -%} 32 | -------------------------------------------------------------------------------- /macros/sql_gen/gen_filters.sql: -------------------------------------------------------------------------------- 1 | {%- macro gen_filters(model_values, start_date, end_date) -%} 2 | {{ return(adapter.dispatch('gen_filters', 'metrics')(model_values, start_date, end_date)) }} 3 | {%- endmacro -%} 4 | 5 | {%- macro default__gen_filters(model_values, start_date, end_date) -%} 6 | 7 | {#- metric start/end dates also applied here to limit incoming data -#} 8 | {% if start_date or end_date %} 9 | and ( 10 | {% if start_date and end_date -%} 11 | cast(base_model.{{model_values.timestamp}} as date) >= cast('{{ start_date }}' as date) 12 | and cast(base_model.{{model_values.timestamp}} as date) <= cast('{{ end_date }}' as date) 13 | {%- elif start_date and not end_date -%} 14 | cast(base_model.{{model_values.timestamp}} as date) >= cast('{{ start_date }}' as date) 15 | {%- elif end_date and not start_date -%} 16 | cast(base_model.{{model_values.timestamp}} as date) <= cast('{{ end_date }}' as date) 17 | {%- endif %} 18 | ) 19 | {% endif -%} 20 | 21 | {#- metric filter clauses... -#} 22 | {% if model_values.filters %} 23 | and ( 24 | {% for filter in model_values.filters -%} 25 | {%- if not loop.first -%} and {% endif %}{{ filter.field }} {{ filter.operator }} {{ filter.value }} 26 | {% endfor -%} 27 | ) 28 | {% endif -%} 29 | 30 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/sql_gen/gen_group_by.sql: -------------------------------------------------------------------------------- 1 | {%- macro gen_group_by(grain, dimensions, calendar_dimensions, relevant_periods) -%} 2 | {{ return(adapter.dispatch('gen_group_by', 'metrics')(grain, dimensions, calendar_dimensions, relevant_periods)) }} 3 | {%- endmacro -%} 4 | 5 | {%- macro default__gen_group_by(grain, dimensions, calendar_dimensions, relevant_periods) -%} 6 | 7 | {#- This model exclusively exists because dynamic group by counts based on range 8 | were too funky when we hardcoded values for 1+1. So we're getting around it by 9 | making it its own function -#} 10 | 11 | {#- The issue arises when we have an initial date column (ie date_month) where month 12 | is also included in the relevent periods. This causes issues and so we need to 13 | remove the grain from the list of relevant periods so it isnt double counted -#} 14 | 15 | {%- set total_dimension_count = metrics.get_total_dimension_count(grain, dimensions, calendar_dimensions, relevant_periods) -%} 16 | 17 | {%- if grain -%} 18 | group by {% for number in range(1,total_dimension_count+1) -%}{{ number }}{%- if not loop.last -%}, {% endif -%} 19 | {%- endfor -%} 20 | {%- else -%} 21 | {%- if total_dimension_count > 0 -%} 22 | group by {% for number in range(1,total_dimension_count+1) -%}{{ number }} {%- if not loop.last -%}, {% endif -%} 23 | {%- endfor -%} 24 | {%- endif -%} 25 | {%- endif -%} 26 | 27 | {%- endmacro -%} 28 | -------------------------------------------------------------------------------- /CHANGELOG.MD: -------------------------------------------------------------------------------- 1 | 9 | 10 | # dbt_metrics v0.1.5 11 | ## What's Changed 12 | * start_date and end_date casting fix by @fivetran-joemarkiewicz in [#22](https://github.com/dbt-labs/dbt_metrics/pull/22) 13 | * Add more integration tests in [#23](https://github.com/dbt-labs/dbt_metrics/pull/23) 14 | 15 | ## New Contributors 16 | * @fivetran-joemarkiewicz made their first contribution in [#22](https://github.com/dbt-labs/dbt_metrics/pull/22) 17 | 18 | **Full Changelog**: https://github.com/dbt-labs/dbt_metrics/compare/0.1.4...0.1.5 19 | 20 | # dbt_metrics v0.1.4 21 | Resolves [#16](https://github.com/dbt-labs/dbt_metrics/issues/16) - period over period calculations were inaccurate when a start date was provided 22 | 23 | # dbt_metrics v0.1.3 24 | - Constrains the date spine to the min/max date range of the metric's model to avoid returning the entire 20 year spine unnecessarily. 25 | - Adds start_date and end_date for manual overrides. An earlier start date than the source model's min value will pull through empty spine rows, a later one will exclude source data. 26 | 27 | # dbt_metrics v0.1.2 28 | Fixes for incorrect constructor in the `rolling` secondary aggregate 29 | 30 | # dbt_metrics v0.1.0 31 | The first release of the dbt_metrics package, which generates queries based on a dbt Core metrics definition. -------------------------------------------------------------------------------- /macros/validation/validate_develop_metrics.sql: -------------------------------------------------------------------------------- 1 | {% macro validate_develop_metrics(metric_list, develop_yml) %} 2 | 3 | {% for metric_name in metric_list %} 4 | {% set metric_definition = develop_yml[metric_name] %} 5 | 6 | {%- if not metric_definition.name %} 7 | {%- do exceptions.raise_compiler_error("The provided yml is missing a metric name") -%} 8 | {%- endif %} 9 | 10 | {%- if not metric_definition.calculation_method %} 11 | {%- do exceptions.raise_compiler_error("The provided yml for metric " ~ metric_definition.name ~ " is missing a calculation method") -%} 12 | {%- endif %} 13 | 14 | {%- if not metric_definition.model and metric_definition.calculation_method != 'derived' %} 15 | {%- do exceptions.raise_compiler_error("The provided yml for metric " ~ metric_definition.name ~ " is missing a model") -%} 16 | {%- endif %} 17 | 18 | {%- if metric_definition.time_grains and grain %} 19 | {%- if grain not in metric_definition.time_grains %} 20 | {%- do exceptions.raise_compiler_error("The selected grain is missing from the metric definition of metric " ~ metric_definition.name ) -%} 21 | {%- endif %} 22 | {%- endif %} 23 | 24 | {%- if not metric_definition.expression %} 25 | {%- do exceptions.raise_compiler_error("The provided yml for metric " ~ metric_definition.name ~ " is missing an expression") -%} 26 | {%- endif %} 27 | 28 | {%- endfor -%} 29 | 30 | {% endmacro %} -------------------------------------------------------------------------------- /.github/workflows/changelog-existence.yml: -------------------------------------------------------------------------------- 1 | # **what?** 2 | # Checks that a file has been committed under the /.changes directory 3 | # as a new CHANGELOG entry. Cannot check for a specific filename as 4 | # it is dynamically generated by change type and timestamp. 5 | # This workflow should not require any secrets since it runs for PRs 6 | # from forked repos. 7 | # By default, secrets are not passed to workflows running from 8 | # a forked repo. 9 | 10 | # **why?** 11 | # Ensure code change gets reflected in the CHANGELOG. 12 | 13 | # **when?** 14 | # This will run for all PRs going into main and *.latest. It will 15 | # run when they are opened, reopened, when any label is added or removed 16 | # and when new code is pushed to the branch. The action will then get 17 | # skipped if the 'Skip Changelog' label is present is any of the labels. 18 | 19 | name: Check Changelog Entry 20 | 21 | on: 22 | pull_request: 23 | types: [opened, reopened, labeled, unlabeled, synchronize] 24 | workflow_dispatch: 25 | 26 | defaults: 27 | run: 28 | shell: bash 29 | 30 | permissions: 31 | contents: read 32 | pull-requests: write 33 | 34 | jobs: 35 | changelog: 36 | uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main 37 | with: 38 | changelog_comment: "Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)." 39 | skip_label: "Skip Changelog" 40 | secrets: inherit -------------------------------------------------------------------------------- /macros/variables/get_base_metrics.sql: -------------------------------------------------------------------------------- 1 | {% macro get_base_metrics(metric) %} 2 | 3 | -- this checks whether it is a relation or a list 4 | {%- if (metric is mapping and metric.get('metadata', {}).get('calculation_method', '').endswith('Relation')) %} 5 | 6 | {%- for child in metric recursive -%} 7 | 8 | {%- if metric.metrics | length > 0 %} 9 | 10 | {# First we get the list of nodes that this metric is dependent on. This is inclusive 11 | of all parent metrics and SHOULD only contain parent metrics #} 12 | {%- set node_list = metric.depends_on.nodes -%} 13 | {%- set metric_list = [] -%} 14 | {# This part is suboptimal - we're looping through the dependent nodes and extracting 15 | the metric name from the idenitfier. Ideally we'd just use the metrics attribute but 16 | right now its a list of lists #} 17 | {%- for node in node_list -%} 18 | {% set metric_name = node.split('.')[2] %} 19 | {% do metric_list.append(metric_name) %} 20 | {%- endfor -%} 21 | {%- endif -%} 22 | {%- endfor -%} 23 | 24 | {% else %} 25 | 26 | {# For non-derived metrics, we just need the relation of the base model ie 27 | the model that its built. Then we append it to the metric list name so the same 28 | variable used in derived metrics can be used below #} 29 | {%- set metric_list = [] -%} 30 | {% do metric_list.append(metric.name) %} 31 | 32 | {%- endif %} 33 | 34 | {% do return(metric_list) %} 35 | 36 | {% endmacro %} -------------------------------------------------------------------------------- /macros/variables/get_faux_metric_tree.sql: -------------------------------------------------------------------------------- 1 | {% macro get_faux_metric_tree(metric_list,develop_yml)%} 2 | 3 | {%- set metric_tree = {'full_set':[]} %} 4 | {%- do metric_tree.update({'parent_set':[]}) -%} 5 | {%- do metric_tree.update({'derived_set':[]}) -%} 6 | {%- do metric_tree.update({'base_set':metric_list}) -%} 7 | {%- do metric_tree.update({'ordered_derived_set':{}}) -%} 8 | 9 | {% for metric_name in metric_list %} 10 | {% set metric_definition = develop_yml[metric_name]%} 11 | {%- set metric_tree = metrics.update_faux_metric_tree(metric_definition, metric_tree, develop_yml) -%} 12 | {% endfor %} 13 | 14 | {%- do metric_tree.update({'full_set':set(metric_tree['full_set'])}) -%} 15 | {%- do metric_tree.update({'parent_set':set(metric_tree['parent_set'])}) -%} 16 | {%- do metric_tree.update({'derived_set':set(metric_tree['derived_set'])}) -%} 17 | 18 | {% for metric_name in metric_tree['parent_set']|unique%} 19 | {%- do metric_tree['ordered_derived_set'].pop(metric_name) -%} 20 | {% endfor %} 21 | 22 | {# This section overrides the derived set by ordering the metrics on their depth so they 23 | can be correctly referenced in the downstream sql query #} 24 | {% set ordered_expression_list = []%} 25 | {% for item in metric_tree['ordered_derived_set']|dictsort(false, 'value') %} 26 | {% if item[0] in metric_tree["derived_set"]%} 27 | {% do ordered_expression_list.append(item[0])%} 28 | {% endif %} 29 | {% endfor %} 30 | {%- do metric_tree.update({'derived_set':ordered_expression_list}) -%} 31 | 32 | {%- do return(metric_tree) -%} 33 | 34 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/models/metric_definitions/base_sum_metric.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | metrics: 3 | - name: base_sum_metric 4 | model: ref('fact_orders') 5 | label: Order Total ($) 6 | timestamp: order_date 7 | time_grains: [day, week, month] 8 | calculation_method: sum 9 | expression: order_total 10 | dimensions: 11 | - had_discount 12 | - order_country 13 | # config: 14 | # restrict_no_time_grain: True 15 | 16 | - name: base_sum_metric_duplicate 17 | model: ref('fact_orders_duplicate') 18 | label: Order Total ($) 19 | timestamp: order_date 20 | time_grains: [day, week, month] 21 | calculation_method: sum 22 | expression: order_total 23 | dimensions: 24 | - had_discount 25 | - order_country 26 | 27 | - name: base_sum_metric__14_day_window 28 | model: ref('fact_orders') 29 | label: Order Total ($) 30 | timestamp: order_date 31 | time_grains: [day, week, month] 32 | calculation_method: sum 33 | expression: order_total 34 | window: 35 | count: 14 36 | period: month 37 | dimensions: 38 | - had_discount 39 | - order_country 40 | 41 | - name: base_test_metric 42 | model: ref('fact_orders') 43 | label: Order Total ($) 44 | timestamp: order_date 45 | time_grains: [day, week, month] 46 | calculation_method: sum 47 | expression: order_total 48 | dimensions: 49 | - had_discount 50 | - order_country 51 | 52 | - name: base_sum_metric__no_timestamp 53 | model: ref('fact_orders') 54 | label: Order Total ($) 55 | calculation_method: sum 56 | expression: order_total 57 | dimensions: 58 | - had_discount 59 | - order_country -------------------------------------------------------------------------------- /macros/secondary_calculations/perform_secondary_calculation.sql: -------------------------------------------------------------------------------- 1 | {%- macro perform_secondary_calculation(metric_name, grain, dimensions, calendar_dimensions, calc_config, metric_config) -%} 2 | {{ return(adapter.dispatch('perform_secondary_calculation', 'metrics')(metric_name, grain, dimensions, calendar_dimensions, calc_config, metric_config)) }} 3 | {%- endmacro -%} 4 | 5 | {% macro default__perform_secondary_calculation(metric_name, grain, dimensions, calendar_dimensions, calc_config, metric_config) %} 6 | {%- set combined_dimensions = dimensions+calendar_dimensions -%} 7 | {%- set calc_type = calc_config.calculation -%} 8 | {%- set calc_sql = '' -%} 9 | 10 | {%- if calc_type == 'period_over_period' -%} 11 | {%- set calc_sql = adapter.dispatch('secondary_calculation_period_over_period', 'metrics')(metric_name, grain, combined_dimensions, calc_config, metric_config) -%} 12 | {%- elif calc_type == 'rolling' -%} 13 | {%- set calc_sql = adapter.dispatch('secondary_calculation_rolling', 'metrics')(metric_name, grain, combined_dimensions, calc_config) -%} 14 | {%- elif calc_type == 'period_to_date' -%} 15 | {%- set calc_sql = adapter.dispatch('secondary_calculation_period_to_date', 'metrics')(metric_name, grain, combined_dimensions, calc_config) -%} 16 | {%- elif calc_type == 'prior' -%} 17 | {%- set calc_sql = adapter.dispatch('secondary_calculation_prior', 'metrics')(metric_name, grain, combined_dimensions, calc_config) -%} 18 | {%- else -%} 19 | {%- do exceptions.raise_compiler_error("Unknown secondary calculation: " ~ calc_type ~ ". calc_config: " ~ calc_config) -%} 20 | {%- endif -%} 21 | {{ calc_sql }} as {{ metrics.generate_secondary_calculation_alias(metric_name, calc_config, grain, true) }} 22 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/secondary_calculations/secondary_calculation_period_over_period.sql: -------------------------------------------------------------------------------- 1 | {%- macro default__secondary_calculation_period_over_period(metric_name, grain, dimensions, calc_config, metric_config) -%} 2 | {%- set calc_sql %} 3 | lag({{ metric_name }}, {{ calc_config.interval }}) over ( 4 | {%- if dimensions %} 5 | partition by {{ dimensions | join(", ") }} 6 | {%- endif %} 7 | order by date_{{grain}} 8 | ) 9 | {%- endset-%} 10 | 11 | {%- if calc_config.comparison_strategy == 'difference' -%} 12 | {% do return (adapter.dispatch('metric_comparison_strategy_difference', 'metrics')(metric_name, calc_sql, metric_config)) %} 13 | 14 | {%- elif calc_config.comparison_strategy == 'ratio' -%} 15 | {% do return (adapter.dispatch('metric_comparison_strategy_ratio', 'metrics')(metric_name, calc_sql, metric_config)) %} 16 | 17 | {%- else -%} 18 | {% do exceptions.raise_compiler_error("Bad comparison_strategy for period_over_period: " ~ calc_config.comparison_strategy ~ ". calc_config: " ~ calc_config) %} 19 | {%- endif -%} 20 | 21 | {% endmacro %} 22 | 23 | {% macro default__metric_comparison_strategy_difference(metric_name, calc_sql, metric_config) -%} 24 | {%- if not metric_config.get("treat_null_values_as_zero", True) %} 25 | {{ metric_name }} - {{ calc_sql }} 26 | {%- else -%} 27 | coalesce({{ metric_name }}, 0) - coalesce({{ calc_sql }}, 0) 28 | {%- endif %} 29 | {%- endmacro -%} 30 | 31 | {% macro default__metric_comparison_strategy_ratio(metric_name, calc_sql, metric_config) -%} 32 | 33 | {%- if not metric_config.get("treat_null_values_as_zero", True) %} 34 | cast({{ metric_name }} as {{ type_float() }}) / nullif({{ calc_sql }}, 0) 35 | {%- else %} 36 | coalesce(cast({{ metric_name }} as {{ type_float() }}) / nullif({{ calc_sql }}, 0) , 0) 37 | {%- endif %} 38 | 39 | {%- endmacro %} 40 | -------------------------------------------------------------------------------- /macros/validation/validate_metric_config.sql: -------------------------------------------------------------------------------- 1 | {%- macro validate_metric_config(metrics_dictionary) -%} 2 | 3 | {#- We loop through the metrics dictionary here to ensure that 4 | 1) all configs are real configs we know about 5 | 2) all of those have valid values passed 6 | returned or used, not just those listed -#} 7 | 8 | {%- set accepted_configs = { 9 | "enabled" : {"accepted_values" : [True, False]}, 10 | "treat_null_values_as_zero" : {"accepted_values" : [True, False]}, 11 | "restrict_no_time_grain" : {"accepted_values" : [True, False]} 12 | } 13 | -%} 14 | 15 | {%- for metric in metrics_dictionary -%} 16 | {%- set metric_config = metrics_dictionary[metric].get("config", none) -%} 17 | {%- if metric_config -%} 18 | {%- for config in metric_config -%} 19 | {%- set config_value = metric_config[config] -%} 20 | {#- some wonkiness here -- metric_config is not a dictionary, it's a MetricConfig object, so can't use the items() method -#} 21 | {#- check that the config is one that we expect -#} 22 | {%- if not accepted_configs[config] -%} 23 | {%- do exceptions.raise_compiler_error("The metric " ~ metric ~ " has an invalid config option. The config '" ~ config ~ "' is not accepted.") -%} 24 | {%- endif -%} 25 | {#- check that the config datatype is expected -#} 26 | {%- if accepted_configs[config] -%} 27 | {%- set accepted_values = accepted_configs[config]["accepted_values"] -%} 28 | {%- if not config_value in accepted_values -%} 29 | {%- do exceptions.raise_compiler_error("The metric " ~ metric ~ " has an invalid config value specified. The config '" ~ config ~ "' expects one of " ~ accepted_values) -%} 30 | {%- endif -%} 31 | {% endif %} 32 | {%- endfor %} 33 | {%- endif -%} 34 | {%- endfor %} 35 | 36 | 37 | 38 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/validation/validate_dimension_list.sql: -------------------------------------------------------------------------------- 1 | {% macro validate_dimension_list(dimensions, metric_tree, metrics_dictionary) %} 2 | 3 | {# This macro exists to invalidate dimensions provided to the metric macro that are not viable 4 | candidates based on metric definitions. This prevents downstream run issues when the sql 5 | logic attempts to group by provided dimensions and fails because they don't exist for 6 | one or more of the required metrics. #} 7 | 8 | {% set calendar_dimensions = var('custom_calendar_dimension_list',[]) %} 9 | 10 | {% for dim in dimensions %} 11 | 12 | {# Now we loop through all the metrics in the full set, which is all metrics, parent metrics, 13 | and derived metrics associated with the macro call #} 14 | {% for metric_name in metric_tree.full_set %} 15 | {% set metric_relation = metrics_dictionary[metric_name]%} 16 | 17 | {# This macro returns a list of dimensions that are inclusive of calendar dimensions #} 18 | {% set complete_dimension_list = metric_relation.dimensions + calendar_dimensions %} 19 | 20 | {# If the dimension provided is not present in the loop metrics dimension list then we 21 | will raise an error. If it is missing in ANY of the metrics, it cannot be used in the 22 | macro call. Only dimensions that are valid in all metrics are valid in the macro call #} 23 | {% if dim not in complete_dimension_list %} 24 | {% if dim not in calendar_dimensions %} 25 | {% do exceptions.raise_compiler_error("The dimension " ~ dim ~ " is not part of the metric " ~ metric_relation.name) %} 26 | {% else %} 27 | {% do exceptions.raise_compiler_error("The dimension " ~ dim ~ " is not part of the metric " ~ metric_relation.name ~ ". If the dimension is from a custom calendar table, please create the custom_calendar_dimension_list as shown in the README.") %} 28 | {% endif %} 29 | {% endif %} 30 | 31 | {%endfor%} 32 | {%endfor%} 33 | 34 | {% endmacro %} -------------------------------------------------------------------------------- /.github/workflows/bot-changelog.yml: -------------------------------------------------------------------------------- 1 | # **what?** 2 | # When bots create a PR, this action will add a corresponding changie yaml file to that 3 | # PR when a specific label is added. 4 | # 5 | # The file is created off a template: 6 | # 7 | # kind: 8 | # body: 9 | # time: 10 | # custom: 11 | # Author: 12 | # Issue: 4904 13 | # PR: 14 | # 15 | # **why?** 16 | # Automate changelog generation for more visability with automated bot PRs. 17 | # 18 | # **when?** 19 | # Once a PR is created, label should be added to PR before or after creation. You can also 20 | # manually trigger this by adding the appropriate label at any time. 21 | # 22 | # **how to add another bot?** 23 | # Add the label and changie kind to the include matrix. That's it! 24 | # 25 | 26 | name: Bot Changelog 27 | 28 | on: 29 | pull_request: 30 | # catch when the PR is opened with the label or when the label is added 31 | types: [labeled] 32 | 33 | permissions: 34 | contents: write 35 | pull-requests: read 36 | 37 | jobs: 38 | generate_changelog: 39 | strategy: 40 | matrix: 41 | include: 42 | - label: "dependencies" 43 | changie_kind: "Dependency" 44 | - label: "snyk" 45 | changie_kind: "Security" 46 | runs-on: ubuntu-latest 47 | 48 | steps: 49 | 50 | - name: Create and commit changelog on bot PR 51 | if: ${{ contains(github.event.pull_request.labels.*.name, matrix.label) }} 52 | id: bot_changelog 53 | uses: emmyoop/changie_bot@22b70618b13d0d1c64ea95212bafca2d2bf6b764 # emmyoop/changie_bot@v1.1.0 54 | with: 55 | GITHUB_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} 56 | commit_author_name: "Github Build Bot" 57 | commit_author_email: "" 58 | commit_message: "Add automated changelog yaml from template for bot PR" 59 | changie_kind: ${{ matrix.changie_kind }} 60 | label: ${{ matrix.label }} 61 | custom_changelog_string: "custom:\n Author: ${{ github.event.pull_request.user.login }}\n Issue: 4904\n PR: ${{ github.event.pull_request.number }}" 62 | -------------------------------------------------------------------------------- /macros/variables/get_metric_definition.sql: -------------------------------------------------------------------------------- 1 | {% macro get_metric_definition(metric_definition) %} 2 | 3 | {% set metrics_dictionary_dict = {} %} 4 | 5 | {% do metrics_dictionary_dict.update({'name': metric_definition.name})%} 6 | {% do metrics_dictionary_dict.update({'calculation_method': metric_definition.calculation_method})%} 7 | {% do metrics_dictionary_dict.update({'timestamp': metric_definition.timestamp})%} 8 | {% do metrics_dictionary_dict.update({'time_grains': metric_definition.time_grains})%} 9 | {% do metrics_dictionary_dict.update({'dimensions': metric_definition.dimensions})%} 10 | {% do metrics_dictionary_dict.update({'filters': metric_definition.filters})%} 11 | {% do metrics_dictionary_dict.update({'config': metric_definition.config})%} 12 | {% if metric_definition.calculation_method != 'derived' %} 13 | {% set metric_model_name = metrics.get_metric_model_name(metric_model=metric_definition.model) %} 14 | {% do metrics_dictionary_dict.update({'metric_model_name': metric_model_name }) %} 15 | {% do metrics_dictionary_dict.update({'metric_model': metrics.get_model_relation(metric_model_name, metric_name)}) %} 16 | {% endif %} 17 | 18 | {# Behavior specific to develop #} 19 | {% if metric_definition is mapping %} 20 | {# We need to do some cleanup for metric parsing #} 21 | {% set metric_expression = metric_definition.expression | replace(" ","") | replace("{{metric('","") | replace("')}}","") | replace("'","") | replace('"',"") %} {% do metrics_dictionary_dict.update({'expression': metric_expression})%} 22 | {% if metric_definition.window %} 23 | {% do metrics_dictionary_dict.update({'window': metric_definition.window}) %} 24 | {% else %} 25 | {% do metrics_dictionary_dict.update({'window': none}) %} 26 | {% endif %} 27 | 28 | {# Behavior specific to calculate #} 29 | {% else %} 30 | {% do metrics_dictionary_dict.update({'expression': metric_definition.expression})%} 31 | {% do metrics_dictionary_dict.update({'window': metric_definition.window})%} 32 | {% endif %} 33 | 34 | {% do return(metrics_dictionary_dict) %} 35 | 36 | {% endmacro %} -------------------------------------------------------------------------------- /macros/sql_gen/build_metric_sql.sql: -------------------------------------------------------------------------------- 1 | {%- macro build_metric_sql(metrics_dictionary, grain, dimensions, secondary_calculations, start_date, end_date, relevant_periods, calendar_dimensions, dimensions_provided, total_dimension_count, group_name, group_values) %} 2 | 3 | {#- This is the SQL Gen part - we've broken each component out into individual macros -#} 4 | {#- We broke this out so it can loop for composite metrics -#} 5 | {{ metrics.gen_aggregate_cte( 6 | metrics_dictionary=metrics_dictionary, 7 | grain=grain, 8 | dimensions=dimensions, 9 | secondary_calculations=secondary_calculations, 10 | start_date=start_date, 11 | end_date=end_date, 12 | relevant_periods=relevant_periods, 13 | calendar_dimensions=calendar_dimensions, 14 | total_dimension_count=total_dimension_count, 15 | group_name=group_name, 16 | group_values=group_values 17 | ) }} 18 | 19 | {#- Diverging path for secondary calcs and needing to datespine -#} 20 | {%- if grain and secondary_calculations | length > 0 -%} 21 | 22 | {%- if dimensions_provided == true -%} 23 | 24 | {{ metrics.gen_dimensions_cte( 25 | group_name=group_name, 26 | dimensions=dimensions 27 | ) }} 28 | 29 | {%- endif -%} 30 | 31 | {{ metrics.gen_spine_time_cte( 32 | group_name=group_name, 33 | grain=grain, 34 | dimensions=dimensions, 35 | secondary_calculations=secondary_calculations, 36 | relevant_periods=relevant_periods, 37 | calendar_dimensions=calendar_dimensions, 38 | dimensions_provided=dimensions_provided 39 | )}} 40 | 41 | {%- endif -%} 42 | 43 | {{ metrics.gen_metric_cte( 44 | metrics_dictionary=metrics_dictionary, 45 | group_name=group_name, 46 | group_values=group_values, 47 | grain=grain, 48 | dimensions=dimensions, 49 | secondary_calculations=secondary_calculations, 50 | start_date=start_date, 51 | end_date=end_date, 52 | relevant_periods=relevant_periods, 53 | calendar_dimensions=calendar_dimensions 54 | )}} 55 | 56 | {%- endmacro -%} 57 | -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_develop_config__missing_timestamp.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/invalid_develop_config.sql 14 | invalid_develop_config_sql = """ 15 | {% set my_metric_yml -%} 16 | 17 | metrics: 18 | - name: invalid_develop_config 19 | model: ref('fact_orders') 20 | label: develop metric dimensions 21 | time_grains: [day, week, month] 22 | calculation_method: sum 23 | expression: order_total 24 | dimensions: 25 | - had_discount 26 | - order_country 27 | 28 | {%- endset %} 29 | 30 | select * 31 | from {{ metrics.develop( 32 | develop_yml=my_metric_yml, 33 | grain='month' 34 | ) 35 | }} 36 | """ 37 | 38 | class TestDevelopMetricDimension: 39 | # configuration in dbt_project.yml 40 | @pytest.fixture(scope="class") 41 | def project_config_update(self): 42 | return { 43 | "name": "example", 44 | "models": {"+materialized": "table"} 45 | } 46 | 47 | # install current repo as package 48 | @pytest.fixture(scope="class") 49 | def packages(self): 50 | return { 51 | "packages": [ 52 | {"local": os.getcwd()} 53 | ] 54 | } 55 | 56 | 57 | # everything that goes in the "seeds" directory 58 | @pytest.fixture(scope="class") 59 | def seeds(self): 60 | return { 61 | "fact_orders_source.csv": fact_orders_source_csv, 62 | } 63 | 64 | # everything that goes in the "models" directory 65 | @pytest.fixture(scope="class") 66 | def models(self): 67 | return { 68 | "fact_orders.sql": fact_orders_sql, 69 | "fact_orders.yml": fact_orders_yml, 70 | "invalid_develop_config.sql": invalid_develop_config_sql, 71 | } 72 | 73 | def test_build_completion(self,project,): 74 | # running deps to install package 75 | results = run_dbt(["deps"]) 76 | results = run_dbt(["seed"]) 77 | 78 | # initial run 79 | results = run_dbt(["run"],expect_pass = False) -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_develop_config__invalid_model.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/invalid_develop_config.sql 14 | invalid_develop_config_sql = """ 15 | {% set my_metric_yml -%} 16 | 17 | metrics: 18 | - name: invalid_develop_config 19 | model: ref('some_unknown_model') 20 | label: develop metric dimensions 21 | timestamp: order_date 22 | time_grains: [day, week, month] 23 | calculation_method: sum 24 | expression: order_total 25 | dimensions: 26 | - had_discount 27 | - order_country 28 | 29 | {%- endset %} 30 | 31 | select * 32 | from {{ metrics.develop( 33 | develop_yml=my_metric_yml, 34 | grain='month' 35 | ) 36 | }} 37 | """ 38 | 39 | class TestDevelopMetricDimension: 40 | # configuration in dbt_project.yml 41 | @pytest.fixture(scope="class") 42 | def project_config_update(self): 43 | return { 44 | "name": "example", 45 | "models": {"+materialized": "table"} 46 | } 47 | 48 | # install current repo as package 49 | @pytest.fixture(scope="class") 50 | def packages(self): 51 | return { 52 | "packages": [ 53 | {"local": os.getcwd()} 54 | ] 55 | } 56 | 57 | 58 | # everything that goes in the "seeds" directory 59 | @pytest.fixture(scope="class") 60 | def seeds(self): 61 | return { 62 | "fact_orders_source.csv": fact_orders_source_csv, 63 | } 64 | 65 | # everything that goes in the "models" directory 66 | @pytest.fixture(scope="class") 67 | def models(self): 68 | return { 69 | "fact_orders.sql": fact_orders_sql, 70 | "fact_orders.yml": fact_orders_yml, 71 | "invalid_develop_config.sql": invalid_develop_config_sql, 72 | } 73 | 74 | def test_build_completion(self,project,): 75 | # running deps to install package 76 | results = run_dbt(["deps"]) 77 | results = run_dbt(["seed"]) 78 | 79 | # initial run 80 | results = run_dbt(["run"],expect_pass = False) -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_develop_config__invalid_type.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/invalid_develop_config.sql 14 | invalid_develop_config_sql = """ 15 | {% set my_metric_yml -%} 16 | 17 | metrics: 18 | - name: invalid_develop_config 19 | model: ref('some_unknown_model') 20 | label: develop metric dimensions 21 | timestamp: order_date 22 | time_grains: [day, week, month] 23 | calculation_method: median 24 | expression: order_total 25 | dimensions: 26 | - had_discount 27 | - order_country 28 | 29 | {%- endset %} 30 | 31 | select * 32 | from {{ metrics.develop( 33 | develop_yml=my_metric_yml, 34 | grain='month' 35 | ) 36 | }} 37 | """ 38 | 39 | class TestDevelopMetricDimension: 40 | # configuration in dbt_project.yml 41 | @pytest.fixture(scope="class") 42 | def project_config_update(self): 43 | return { 44 | "name": "example", 45 | "models": {"+materialized": "table"} 46 | } 47 | 48 | # install current repo as package 49 | @pytest.fixture(scope="class") 50 | def packages(self): 51 | return { 52 | "packages": [ 53 | {"local": os.getcwd()} 54 | ] 55 | } 56 | 57 | 58 | # everything that goes in the "seeds" directory 59 | @pytest.fixture(scope="class") 60 | def seeds(self): 61 | return { 62 | "fact_orders_source.csv": fact_orders_source_csv, 63 | } 64 | 65 | # everything that goes in the "models" directory 66 | @pytest.fixture(scope="class") 67 | def models(self): 68 | return { 69 | "fact_orders.sql": fact_orders_sql, 70 | "fact_orders.yml": fact_orders_yml, 71 | "invalid_develop_config.sql": invalid_develop_config_sql, 72 | } 73 | 74 | def test_build_completion(self,project,): 75 | # running deps to install package 76 | results = run_dbt(["deps"]) 77 | results = run_dbt(["seed"]) 78 | 79 | # initial run 80 | results = run_dbt(["run"],expect_pass = False) -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_develop_config_invalid_model.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/invalid_develop_config.sql 14 | invalid_develop_config_sql = """ 15 | {% set my_metric_yml -%} 16 | 17 | metrics: 18 | - name: invalid_develop_config 19 | model: ref('some_unknown_model') 20 | label: develop metric dimensions 21 | timestamp: order_date 22 | time_grains: [day, week, month] 23 | calculation_method: sum 24 | expression: order_total 25 | dimensions: 26 | - had_discount 27 | - order_country 28 | 29 | {%- endset %} 30 | 31 | select * 32 | from {{ metrics.develop( 33 | develop_yml=my_metric_yml, 34 | grain='month' 35 | ) 36 | }} 37 | """ 38 | 39 | class TestDevelopMetricDimension: 40 | # configuration in dbt_project.yml 41 | @pytest.fixture(scope="class") 42 | def project_config_update(self): 43 | return { 44 | "name": "example", 45 | "models": {"+materialized": "table"} 46 | } 47 | 48 | # install current repo as package 49 | @pytest.fixture(scope="class") 50 | def packages(self): 51 | return { 52 | "packages": [ 53 | {"local": os.getcwd()} 54 | ] 55 | } 56 | 57 | 58 | # everything that goes in the "seeds" directory 59 | @pytest.fixture(scope="class") 60 | def seeds(self): 61 | return { 62 | "fact_orders_source.csv": fact_orders_source_csv, 63 | } 64 | 65 | # everything that goes in the "models" directory 66 | @pytest.fixture(scope="class") 67 | def models(self): 68 | return { 69 | "fact_orders.sql": fact_orders_sql, 70 | "fact_orders.yml": fact_orders_yml, 71 | "invalid_develop_config.sql": invalid_develop_config_sql, 72 | } 73 | 74 | def test_build_completion(self,project,): 75 | # running deps to install package 76 | results = run_dbt(["deps"]) 77 | results = run_dbt(["seed"]) 78 | 79 | # initial run 80 | results = run_dbt(["run"],expect_pass = False) 81 | -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_date_datatype.py: -------------------------------------------------------------------------------- 1 | from configparser import ParsingError 2 | from struct import pack 3 | import os 4 | import pytest 5 | from dbt.tests.util import run_dbt 6 | 7 | # our file contents 8 | from tests.functional.fixtures import ( 9 | fact_orders_source_csv, 10 | fact_orders_sql, 11 | fact_orders_yml, 12 | ) 13 | 14 | # models/max_date_invalid_datatype.sql 15 | max_date_invalid_datatype_sql = """ 16 | select * 17 | from 18 | {{ metrics.calculate(metric('max_date_invalid_datatype'), 19 | grain='day' 20 | ) 21 | }} 22 | """ 23 | 24 | # models/invalid_metric_names.yml 25 | invalid_metric_names_yml = """ 26 | version: 2 27 | 28 | metrics: 29 | - name: max_date_invalid_datatype 30 | model: ref('fact_orders') 31 | label: max_date_invalid_datatype 32 | timestamp: order_date 33 | time_grains: [day, week, month] 34 | calculation_method: max 35 | expression: order_date 36 | """ 37 | 38 | class TestInvalidDatatypes: 39 | 40 | # configuration in dbt_project.yml 41 | @pytest.fixture(scope="class") 42 | def project_config_update(self): 43 | return { 44 | "name": "example", 45 | "models": {"+materialized": "table"} 46 | } 47 | 48 | # install current repo as package 49 | @pytest.fixture(scope="class") 50 | def packages(self): 51 | return { 52 | "packages": [ 53 | {"local": os.getcwd()} 54 | ] 55 | } 56 | 57 | 58 | # everything that goes in the "seeds" directory 59 | @pytest.fixture(scope="class") 60 | def seeds(self): 61 | return { 62 | "fact_orders_source.csv": fact_orders_source_csv 63 | } 64 | 65 | # everything that goes in the "models" directory 66 | @pytest.fixture(scope="class") 67 | def models(self): 68 | return { 69 | "fact_orders.sql": fact_orders_sql, 70 | "fact_orders.yml": fact_orders_yml, 71 | "max_date_invalid_datatype.sql": max_date_invalid_datatype_sql, 72 | "invalid_metric_names.yml": invalid_metric_names_yml 73 | } 74 | 75 | def test_invalid_date_datatype(self,project,): 76 | # running deps to install package 77 | run_dbt(["deps"]) 78 | run_dbt(["seed"]) 79 | 80 | # initial run 81 | run_dbt(["run"],expect_pass = False) -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_develop_config__invalid_dimension.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/invalid_develop_config.sql 14 | invalid_develop_config_sql = """ 15 | {% set my_metric_yml -%} 16 | 17 | metrics: 18 | - name: invalid_develop_config 19 | model: ref('fact_orders') 20 | label: develop metric dimensions 21 | timestamp: order_date 22 | time_grains: [day, week, month] 23 | calculation_method: sum 24 | expression: order_total 25 | dimensions: 26 | - had_discount 27 | - order_country 28 | 29 | {%- endset %} 30 | 31 | select * 32 | from {{ metrics.develop( 33 | develop_yml=my_metric_yml, 34 | grain='month', 35 | dimensions=['invalid_dimension_name'] 36 | ) 37 | }} 38 | """ 39 | 40 | class TestDevelopMetricDimension: 41 | # configuration in dbt_project.yml 42 | @pytest.fixture(scope="class") 43 | def project_config_update(self): 44 | return { 45 | "name": "example", 46 | "models": {"+materialized": "table"} 47 | } 48 | 49 | # install current repo as package 50 | @pytest.fixture(scope="class") 51 | def packages(self): 52 | return { 53 | "packages": [ 54 | {"local": os.getcwd()} 55 | ] 56 | } 57 | 58 | 59 | # everything that goes in the "seeds" directory 60 | @pytest.fixture(scope="class") 61 | def seeds(self): 62 | return { 63 | "fact_orders_source.csv": fact_orders_source_csv, 64 | } 65 | 66 | # everything that goes in the "models" directory 67 | @pytest.fixture(scope="class") 68 | def models(self): 69 | return { 70 | "fact_orders.sql": fact_orders_sql, 71 | "fact_orders.yml": fact_orders_yml, 72 | "invalid_develop_config.sql": invalid_develop_config_sql, 73 | } 74 | 75 | def test_build_completion(self,project,): 76 | # running deps to install package 77 | results = run_dbt(["deps"]) 78 | results = run_dbt(["seed"]) 79 | 80 | # initial run 81 | results = run_dbt(["run"],expect_pass = False) -------------------------------------------------------------------------------- /macros/variables/get_models_grouping.sql: -------------------------------------------------------------------------------- 1 | {%- macro get_models_grouping(metric_tree, metrics_dictionary) -%} 2 | {#- 3 | The purpose of this macro is to create a dictionary that can be used by 4 | gen_base_query and gen_aggregate_query in order to intelligently group 5 | metrics together on whether they can be queried in the same query. These 6 | will be grouped together with a unique model name as the key and the value 7 | containing the list of the metrics. This is complicated because we allow 8 | different properties that affect the base query, so we can't do a single 9 | grouping based on model. As such, if a metric contains one of these properties 10 | we have to create a group for that specific combination. 11 | 12 | The properties that cause us to group the metric seperately are: 13 | - windows 14 | - filters 15 | - timestamp fields 16 | 17 | In order to ensure consistency, we will also include those values in the 18 | dictionary so we can reference them from the metrics grouping (ie a single 19 | location) instead of from a randomly selected metric in the list of metrics. 20 | 21 | An example output looks like: 22 | { 23 | 'model_4f977327f02b5c04af4337f54ed81a17': { 24 | 'metric_names':['metric_a','metric_b'], 25 | 'metric_timestamp': order_date, 26 | 'metric_filters':[ 27 | MetricFilter(field='had_discount', operator='is', value='true'), 28 | MetricFilter(field='order_country', operator='=', value='CA') 29 | ] 30 | 'metric_window': MetricTime(count=14, period=) 31 | } 32 | } 33 | -#} 34 | 35 | {% set models_grouping = {} %} 36 | 37 | {% for metric_name in metric_tree.parent_set %} 38 | {% set metric_dictionary = metrics_dictionary[metric_name] %} 39 | 40 | {% set models_grouping = metrics.get_model_group( 41 | models_grouping=models_grouping, 42 | metric_model=metric_dictionary.metric_model, 43 | metric_model_name=metric_dictionary.metric_model_name, 44 | metric_name=metric_dictionary.name, 45 | metric_timestamp=metric_dictionary.timestamp, 46 | metric_filters=metric_dictionary.filters, 47 | metric_window=metric_dictionary.window 48 | ) %} 49 | 50 | {% endfor %} 51 | 52 | {% do return(models_grouping) %} 53 | 54 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/sql_gen/gen_calendar_join.sql: -------------------------------------------------------------------------------- 1 | {% macro gen_calendar_join(group_values) %} 2 | {{ return(adapter.dispatch('gen_calendar_join', 'metrics')(group_values)) }} 3 | {%- endmacro -%} 4 | 5 | {% macro default__gen_calendar_join(group_values) %} 6 | left join calendar 7 | {%- if group_values.window is not none %} 8 | on cast(base_model.{{group_values.timestamp}} as date) > dateadd({{group_values.window.period}}, -{{group_values.window.count}}, calendar.date_day) 9 | and cast(base_model.{{group_values.timestamp}} as date) <= calendar.date_day 10 | {%- else %} 11 | on cast(base_model.{{group_values.timestamp}} as date) = calendar.date_day 12 | {% endif -%} 13 | {% endmacro %} 14 | 15 | {% macro bigquery__gen_calendar_join(group_values) %} 16 | left join calendar 17 | {%- if group_values.window is not none %} 18 | on cast(base_model.{{group_values.timestamp}} as date) > date_sub(calendar.date_day, interval {{group_values.window.count}} {{group_values.window.period}}) 19 | and cast(base_model.{{group_values.timestamp}} as date) <= calendar.date_day 20 | {%- else %} 21 | on cast(base_model.{{group_values.timestamp}} as date) = calendar.date_day 22 | {% endif -%} 23 | {% endmacro %} 24 | 25 | {% macro postgres__gen_calendar_join(group_values) %} 26 | left join calendar 27 | {%- if group_values.window is not none %} 28 | on cast(base_model.{{group_values.timestamp}} as date) > calendar.date_day - interval '{{group_values.window.count}} {{group_values.window.period}}' 29 | and cast(base_model.{{group_values.timestamp}} as date) <= calendar.date_day 30 | {%- else %} 31 | on cast(base_model.{{group_values.timestamp}} as date) = calendar.date_day 32 | {% endif -%} 33 | {% endmacro %} 34 | 35 | {% macro redshift__gen_calendar_join(group_values) %} 36 | left join calendar 37 | {%- if group_values.window is not none %} 38 | on cast(base_model.{{group_values.timestamp}} as date) > dateadd({{group_values.window.period}}, -{{group_values.window.count}}, calendar.date_day) 39 | and cast(base_model.{{group_values.timestamp}} as date) <= calendar.date_day 40 | {%- else %} 41 | on cast(base_model.{{group_values.timestamp}} as date) = calendar.date_day 42 | {% endif -%} 43 | {% endmacro %} 44 | -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_undefined_metric.py: -------------------------------------------------------------------------------- 1 | from configparser import ParsingError 2 | from struct import pack 3 | import os 4 | import pytest 5 | from dbt.tests.util import run_dbt 6 | from dbt.exceptions import TargetNotFoundError 7 | 8 | # our file contents 9 | from tests.functional.fixtures import ( 10 | fact_orders_source_csv, 11 | fact_orders_sql, 12 | fact_orders_yml, 13 | ) 14 | 15 | # models/undefined_metric.sql 16 | undefined_metric_sql = """ 17 | select * 18 | from 19 | {{ metrics.calculate(metric('undefined_metric'), 20 | grain='month' 21 | ) 22 | }} 23 | """ 24 | 25 | # models/undefined_metric.yml 26 | undefined_metric_yml = """ 27 | version: 2 28 | models: 29 | - name: undefined_metric 30 | 31 | metrics: 32 | - name: not_undefined_metric 33 | model: ref('fact_orders') 34 | label: Total Discount ($) 35 | timestamp: order_date 36 | time_grains: [day, week, month] 37 | calculation_method: count 38 | expression: order_total 39 | dimensions: 40 | - had_discount 41 | - order_country 42 | """ 43 | 44 | class TestUndefinedMetric: 45 | 46 | # configuration in dbt_project.yml 47 | @pytest.fixture(scope="class") 48 | def project_config_update(self): 49 | return { 50 | "name": "example", 51 | "models": {"+materialized": "table"} 52 | } 53 | 54 | # install current repo as package 55 | @pytest.fixture(scope="class") 56 | def packages(self): 57 | return { 58 | "packages": [ 59 | {"local": os.getcwd()} 60 | ] 61 | } 62 | 63 | 64 | # everything that goes in the "seeds" directory 65 | @pytest.fixture(scope="class") 66 | def seeds(self): 67 | return { 68 | "fact_orders_source.csv": fact_orders_source_csv 69 | } 70 | 71 | # everything that goes in the "models" directory 72 | @pytest.fixture(scope="class") 73 | def models(self): 74 | return { 75 | "fact_orders.sql": fact_orders_sql, 76 | "fact_orders.yml": fact_orders_yml, 77 | "undefined_metric.sql": undefined_metric_sql, 78 | "undefined_metric.yml": undefined_metric_yml 79 | } 80 | 81 | def test_undefined_metric(self,project,): 82 | with pytest.raises(TargetNotFoundError): 83 | run_dbt(["deps"]) 84 | run_dbt(["seed"]) 85 | run_dbt(["run"]) -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_metric_name.py: -------------------------------------------------------------------------------- 1 | from configparser import ParsingError 2 | from struct import pack 3 | import os 4 | import pytest 5 | from dbt.tests.util import run_dbt 6 | from dbt.exceptions import ParsingError 7 | 8 | # our file contents 9 | from tests.functional.fixtures import ( 10 | fact_orders_source_csv, 11 | fact_orders_sql, 12 | fact_orders_yml, 13 | ) 14 | 15 | # models/invalid_metric_name.sql 16 | invalid_metric_name_sql = """ 17 | select * 18 | from 19 | {{ metrics.calculate(metric('invalid metric name'), 20 | grain='month' 21 | ) 22 | }} 23 | """ 24 | 25 | # models/invalid_metric_name.yml 26 | invalid_metric_name_yml = """ 27 | version: 2 28 | models: 29 | - name: invalid_metric_name 30 | 31 | metrics: 32 | - name: invalid metric name 33 | model: ref('fact_orders') 34 | label: Total Discount ($) 35 | timestamp: order_date 36 | time_grains: [day, week, month] 37 | calculation_method: count 38 | expression: order_total 39 | dimensions: 40 | - had_discount 41 | - order_country 42 | """ 43 | 44 | class TestInvalidMetricName: 45 | 46 | # configuration in dbt_project.yml 47 | @pytest.fixture(scope="class") 48 | def project_config_update(self): 49 | return { 50 | "name": "example", 51 | "models": {"+materialized": "table"} 52 | } 53 | 54 | # install current repo as package 55 | @pytest.fixture(scope="class") 56 | def packages(self): 57 | return { 58 | "packages": [ 59 | {"local": os.getcwd()} 60 | ] 61 | } 62 | 63 | 64 | # everything that goes in the "seeds" directory 65 | @pytest.fixture(scope="class") 66 | def seeds(self): 67 | return { 68 | "fact_orders_source.csv": fact_orders_source_csv 69 | } 70 | 71 | # everything that goes in the "models" directory 72 | @pytest.fixture(scope="class") 73 | def models(self): 74 | return { 75 | "fact_orders.sql": fact_orders_sql, 76 | "fact_orders.yml": fact_orders_yml, 77 | "invalid_metric_name.sql": invalid_metric_name_sql, 78 | "invalid_metric_name.yml": invalid_metric_name_yml 79 | } 80 | 81 | def test_model_name(self,project,): 82 | # initial run 83 | with pytest.raises(ParsingError): 84 | run_dbt(["deps"]) 85 | run_dbt(["run"]) -------------------------------------------------------------------------------- /integration_tests/macros/notes.sql: -------------------------------------------------------------------------------- 1 | {# Now we see if the node already exists in the metric tree and return that if 2 | it does so that we're not creating duplicates #} 3 | {# {%- if metric_tree[node.unique_id] is defined -%} 4 | 5 | {% do return(metric_tree[node.unique_id]) -%} 6 | 7 | {%- endif -%} 8 | 9 | {# {{ log("Inside Macro Depends on: " ~ node.depends_on, info=true) }} #} 10 | 11 | 12 | {# Here we create two sets, sets being the same as lists but they account for uniqueness. 13 | One is the full set, which contains all of the parent metrics and the other is the leaf 14 | set, which we'll use to determine the leaf, or base metrics. #} 15 | {%- set full_set = [] -%} 16 | {%- set leaf_set = [] -%} 17 | 18 | {# We define parent nodes as being the parent nodes that begin with metric, which lets 19 | us filter out model nodes #} 20 | {%- set parent_nodes = node.depends_on.nodes -%} 21 | 22 | {%- for parent_node in parent_nodes -%} 23 | 24 | {# We set an if condition based on if parent nodes. If there are none, then this metric 25 | is a leaf node and any recursive loop should end #} 26 | {%- if parent_nodes -%} 27 | 28 | {# Now we finally recurse through the nodes. We begin by filtering the overall list we 29 | recurse through by limiting it to depending on metric nodes and not ALL nodes #} 30 | {%- for parent_id in parent_nodes -%} 31 | 32 | {# Then we add the parent_id of the metric to the full set. If it already existed 33 | then it won't make an impact but we want to make sure it is represented #} 34 | {%- do full_set.update(parent_id) -%} 35 | 36 | {# And here we re-run the current macro but fill in the parent_id so that we loop again 37 | with that metric information. You may be wondering, why are you using parent_id? Doesn't 38 | the DAG always go from parent to child? Normally, yes! With this, no! We're reversing the 39 | DAG and going up to parents to find the leaf nodes that are really parent nodes. #} 40 | {%- set new_parent = metrics_list[parent_id] -%} 41 | {%- do full_set.update(get_metric_parents(new_parent,metrics_list,metric_tree)) -%} 42 | 43 | {%- endfor -%} 44 | 45 | {%- else -%} 46 | 47 | {%- do leaf_set.update(node.unique_id) -%} 48 | 49 | {%- endif -%} 50 | 51 | {%- endfor -%} 52 | 53 | {%- do return(full_set) -%} #} -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_derived_metric.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/invalid_derived_metric.sql 14 | invalid_derived_metric_sql = """ 15 | select * 16 | from 17 | {{ metrics.calculate(metric('invalid_derived_metric'), 18 | grain='month' 19 | ) 20 | }} 21 | """ 22 | 23 | # models/invalid_derived_metric.yml 24 | invalid_derived_metric_yml = """ 25 | version: 2 26 | models: 27 | - name: invalid_derived_metric 28 | 29 | metrics: 30 | - name: invalid_derived_metric 31 | label: Total Discount ($) 32 | timestamp: order_date 33 | time_grains: [day, week, month] 34 | calculation_method: derived 35 | expression: order_total 36 | dimensions: 37 | - had_discount 38 | - order_country 39 | """ 40 | 41 | class TestInvalidDerivedMetric: 42 | 43 | # configuration in dbt_project.yml 44 | @pytest.fixture(scope="class") 45 | def project_config_update(self): 46 | return { 47 | "name": "example", 48 | "models": {"+materialized": "table"} 49 | } 50 | 51 | # install current repo as package 52 | @pytest.fixture(scope="class") 53 | def packages(self): 54 | return { 55 | "packages": [ 56 | {"local": os.getcwd()} 57 | ] 58 | } 59 | 60 | 61 | # everything that goes in the "seeds" directory 62 | @pytest.fixture(scope="class") 63 | def seeds(self): 64 | return { 65 | "fact_orders_source.csv": fact_orders_source_csv 66 | } 67 | 68 | # everything that goes in the "models" directory 69 | @pytest.fixture(scope="class") 70 | def models(self): 71 | return { 72 | "fact_orders.sql": fact_orders_sql, 73 | "fact_orders.yml": fact_orders_yml, 74 | "invalid_derived_metric.sql": invalid_derived_metric_sql, 75 | "invalid_derived_metric.yml": invalid_derived_metric_yml 76 | } 77 | 78 | def test_invalid_derived_metric(self,project,): 79 | # running deps to install package 80 | results = run_dbt(["deps"]) 81 | 82 | # seed seeds 83 | results = run_dbt(["seed"]) 84 | assert len(results) == 1 85 | 86 | # Here we expect the run to fail because the incorrect 87 | # config won't allow it to compile 88 | run_dbt(["run"], expect_pass = False) -------------------------------------------------------------------------------- /macros/sql_gen/gen_base_query.sql: -------------------------------------------------------------------------------- 1 | {% macro gen_base_query(metrics_dictionary, grain, dimensions, secondary_calculations, start_date, end_date, relevant_periods, calendar_dimensions, total_dimension_count, group_name, group_values) %} 2 | {{ return(adapter.dispatch('gen_base_query', 'metrics')(metrics_dictionary, grain, dimensions, secondary_calculations, start_date, end_date, relevant_periods, calendar_dimensions, total_dimension_count, group_name, group_values)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__gen_base_query(metrics_dictionary, grain, dimensions, secondary_calculations, start_date, end_date, relevant_periods, calendar_dimensions, total_dimension_count, group_name, group_values) %} 6 | {# This is the "base" CTE which selects the fields we need to correctly 7 | calculate the metric. -#} 8 | select 9 | {% if grain -%} 10 | {#- 11 | Given that we've already determined the metrics in metric_names share 12 | the same windows & filters, we can base the conditional off of the first 13 | value in the list because the order doesn't matter. 14 | -#} 15 | cast(base_model.{{group_values.timestamp}} as date) as metric_date_day, 16 | calendar.date_{{ grain }} as date_{{grain}}, 17 | calendar.date_day as window_filter_date, 18 | {%- if secondary_calculations | length > 0 %} 19 | {%- for period in relevant_periods %} 20 | calendar.date_{{ period }}, 21 | {%- endfor -%} 22 | {%- endif -%} 23 | {%- endif -%} 24 | {#- -#} 25 | {%- for dim in dimensions %} 26 | base_model.{{ dim }}, 27 | {%- endfor %} 28 | {%- for calendar_dim in calendar_dimensions -%} 29 | calendar.{{ calendar_dim }}, 30 | {%- endfor -%} 31 | {%- for metric_name in group_values.metric_names -%} 32 | {{ metrics.gen_property_to_aggregate(metrics_dictionary[metric_name], grain, dimensions, calendar_dimensions) }} 33 | {%- if not loop.last -%},{%- endif -%} 34 | {%- endfor%} 35 | from {{ group_values.metric_model }} base_model 36 | {# -#} 37 | {%- if grain or calendar_dimensions|length > 0 -%} 38 | {{ metrics.gen_calendar_join(group_values) }} 39 | {%- endif -%} 40 | {# #} 41 | where 1=1 42 | {#- -#} 43 | {{ metrics.gen_filters(group_values, start_date, end_date) }} 44 | {# #} 45 | 46 | {%- endmacro -%} -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_string_datatype.py: -------------------------------------------------------------------------------- 1 | from configparser import ParsingError 2 | from struct import pack 3 | import os 4 | import pytest 5 | from dbt.tests.util import run_dbt 6 | 7 | # our file contents 8 | from tests.functional.fixtures import ( 9 | fact_orders_source_csv, 10 | fact_orders_sql, 11 | fact_orders_yml, 12 | ) 13 | 14 | # models/max_string_invalid_datatype.sql 15 | max_string_invalid_datatype_sql = """ 16 | select * 17 | from 18 | {{ metrics.calculate(metric('max_string_invalid_datatype'), 19 | grain='day' 20 | ) 21 | }} 22 | """ 23 | 24 | # models/invalid_metric_names.yml 25 | invalid_metric_names_yml = """ 26 | version: 2 27 | 28 | metrics: 29 | - name: max_string_invalid_datatype 30 | model: ref('fact_orders') 31 | label: max_date_invalid_datatype 32 | timestamp: order_date 33 | time_grains: [day, week, month] 34 | calculation_method: max 35 | expression: order_country 36 | """ 37 | 38 | class TestInvalidStringDataType: 39 | 40 | # configuration in dbt_project.yml 41 | @pytest.fixture(scope="class") 42 | def project_config_update(self): 43 | return { 44 | "name": "example", 45 | "models": {"+materialized": "table"} 46 | } 47 | 48 | # install current repo as package 49 | @pytest.fixture(scope="class") 50 | def packages(self): 51 | return { 52 | "packages": [ 53 | {"local": os.getcwd()} 54 | ] 55 | } 56 | 57 | 58 | # everything that goes in the "seeds" directory 59 | @pytest.fixture(scope="class") 60 | def seeds(self): 61 | return { 62 | "fact_orders_source.csv": fact_orders_source_csv 63 | } 64 | 65 | # everything that goes in the "models" directory 66 | @pytest.fixture(scope="class") 67 | def models(self): 68 | return { 69 | "fact_orders.sql": fact_orders_sql, 70 | "fact_orders.yml": fact_orders_yml, 71 | "max_string_invalid_datatype.sql":max_string_invalid_datatype_sql, 72 | "invalid_metric_names.yml": invalid_metric_names_yml 73 | } 74 | 75 | def test_build_completion(self,project,): 76 | # running deps to install package 77 | results = run_dbt(["deps"]) 78 | results = run_dbt(["seed"]) 79 | 80 | if os.getenv('dbt_target') == 'databricks': 81 | # initial run. Databricks has a funky way of handling coalesce 82 | results = run_dbt(["run"]) 83 | else: 84 | # initial run 85 | results = run_dbt(["run"], expect_pass = False) 86 | -------------------------------------------------------------------------------- /macros/misc/metrics__date_spine.sql: -------------------------------------------------------------------------------- 1 | {% macro metric_date_spine(datepart, start_date, end_date) %} 2 | {{ return(adapter.dispatch('metric_date_spine', 'metrics')(datepart, start_date, end_date)) }} 3 | {%- endmacro %} 4 | 5 | {% macro default__metric_date_spine(datepart, start_date, end_date) %} 6 | 7 | 8 | {# call as follows: 9 | 10 | metric_date_spine( 11 | "day", 12 | "to_date('01/01/2016', 'mm/dd/yyyy')", 13 | "dateadd(week, 1, current_date)" 14 | ) #} 15 | 16 | 17 | with rawdata as ( 18 | 19 | {{metrics.metric_generate_series(14610)}} 20 | 21 | ), 22 | 23 | all_periods as ( 24 | 25 | select ( 26 | {{ 27 | dateadd( 28 | datepart, 29 | "row_number() over (order by 1) - 1", 30 | start_date 31 | ) 32 | }} 33 | ) as date_{{datepart}} 34 | from rawdata 35 | 36 | ), 37 | 38 | filtered as ( 39 | 40 | select * 41 | from all_periods 42 | where date_{{datepart}} <= {{ end_date }} 43 | 44 | ) 45 | 46 | select * from filtered 47 | 48 | {% endmacro %} 49 | 50 | 51 | {% macro metric_get_powers_of_two(upper_bound) %} 52 | {{ return(adapter.dispatch('metric_get_powers_of_two', 'metrics')(upper_bound)) }} 53 | {% endmacro %} 54 | 55 | {% macro default__metric_get_powers_of_two(upper_bound) %} 56 | 57 | {% if upper_bound <= 0 %} 58 | {{ exceptions.raise_compiler_error("upper bound must be positive") }} 59 | {% endif %} 60 | 61 | {% for _ in range(1, 100) %} 62 | {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} 63 | {% endfor %} 64 | 65 | {% endmacro %} 66 | 67 | 68 | {% macro metric_generate_series(upper_bound) %} 69 | {{ return(adapter.dispatch('metric_generate_series', 'metrics')(upper_bound)) }} 70 | {% endmacro %} 71 | 72 | {% macro default__metric_generate_series(upper_bound) %} 73 | 74 | {% set n = metrics.metric_get_powers_of_two(upper_bound) %} 75 | 76 | with p as ( 77 | select 0 as generated_number union all select 1 78 | ), unioned as ( 79 | 80 | select 81 | 82 | {% for i in range(n) %} 83 | p{{i}}.generated_number * power(2, {{i}}) 84 | {% if not loop.last %} + {% endif %} 85 | {% endfor %} 86 | + 1 87 | as generated_number 88 | 89 | from 90 | 91 | {% for i in range(n) %} 92 | p as p{{i}} 93 | {% if not loop.last %} cross join {% endif %} 94 | {% endfor %} 95 | 96 | ) 97 | 98 | select * 99 | from unioned 100 | where generated_number <= {{upper_bound}} 101 | order by generated_number 102 | 103 | {% endmacro %} -------------------------------------------------------------------------------- /macros/README.md: -------------------------------------------------------------------------------- 1 | ### Understanding metrics macros 2 | If you're interested in writing your own metrics macro or are curious about how the sql is generated for your metrics query, you've come to the right place! This readme will contain information on the flow of the most important macros, the inputs needed to make them work, and short explanations of how they all work! 3 | 4 | #### The Flow 5 | As of version v0.3.2, significant work has been done on breaking out logic into discrete and logical components to ensure that each macro always performs the same behavior, regardless of how or where it is called. To wit, the first metrics always called are either: 6 | 7 | - **calculate**: this is the most frequently used macro by end-users and is documented well in the overarching README. 8 | - **develop**: this macro allows users to provide metric yml and test/simulate what the end result would look like if said metric were included in their project 9 | 10 | Once these macros are called, they both go through two logical steps albeit in slightly different ways. 11 | 12 | - validation: Both macros validate that their inputs are correct and match what we are expecting to see. Additionally they also validate the inputs against the existing metrics object in the manifest to ensure that dimensions are correct, time grains are permitted, etc etc. 13 | - variable creation: Both macros also create 2 variables that are required in downstream processes. The `metric_tree` and the `metrics_dictionary`. 14 | 15 | **Metric Tree**: This object is a dictionary that contains 5 key value pairs: 16 | - full_set: this value is a list of **all** metric names that are required to construct the sql. It includes all parent metrics and experssion metrics. 17 | - base_set: this value is a list of metric names that are provided to the macro. 18 | - parent_set: this value is a list of parent metric names, which are defined as all first level (non-derived) metrics upon which downstream metrics are dependent upon. 19 | - derived_set: this value is a list of derived metric names 20 | - ordered_derived_set: this value is a list of dictionaries that contains the derived metrics **and** their depth from the parent. This is used to construct the nested CTEs in the sql gen. 21 | 22 | **Metrics Dictionary**: This object is a dictionary that contains all of the attributes for each metric in the full_set. It was implemented in v0.3.2 to support the same input provided to `get_metric_sql` from both develop and calculate. It must contain the: 23 | - Metric name 24 | - Metric calculation method 25 | - Metric expression 26 | - Metric timestamp 27 | - Metric time grains 28 | - Metric dimensions 29 | - Metric filters 30 | - (If not derived) Metric model name 31 | - (If not derived) Metric model object -------------------------------------------------------------------------------- /macros/validation/validate_grain.sql: -------------------------------------------------------------------------------- 1 | {% macro validate_grain(grain, metric_tree, metrics_dictionary, secondary_calculations, dimensions) %} 2 | 3 | {# We loop through the full set here to ensure that the provided grain works for all metrics 4 | returned or used, not just those listed #} 5 | {% if grain %} 6 | {%- if not grain and secondary_calculations | length > 0 -%} 7 | {%- do exceptions.raise_compiler_error("Secondary calculations require a grain to be provided") -%} 8 | {%- endif -%} 9 | 10 | 11 | {% for metric_name in metric_tree.full_set %} 12 | {% set metric_relation = metrics_dictionary[metric_name]%} 13 | 14 | {% if grain not in metric_relation.time_grains%} 15 | {% if metric_name not in metric_tree.base_set %} 16 | {%- do exceptions.raise_compiler_error("The metric " ~ metric_name ~ " is an upstream metric of one of the provided metrics. The grain " ~ grain ~ " is not defined in its metric definition.") %} 17 | {% else %} 18 | {%- do exceptions.raise_compiler_error("The metric " ~ metric_name ~ " does not have the provided time grain " ~ grain ~ " defined in the metric definition.") %} 19 | {% endif %} 20 | {% endif %} 21 | {% endfor %} 22 | 23 | {% elif not grain %} 24 | {% for metric_name in metric_tree.full_set %} 25 | {% set metric_relation = metrics_dictionary[metric_name]%} 26 | {% if metric_relation.get("config").get("restrict_no_time_grain", False) == True %} 27 | {% if metric_name not in metric_tree.base_set %} 28 | {%- do exceptions.raise_compiler_error("The metric " ~ metric_relation.name ~ " is an upstream metric of one of the provided metrics and has been configured to not allow non time-grain queries.") %} 29 | {% else %} 30 | {%- do exceptions.raise_compiler_error("The metric " ~ metric_relation.name ~ " has been configured to not allow non time-grain queries.") %} 31 | {% endif %} 32 | {% endif %} 33 | 34 | {% endfor %} 35 | 36 | {% if secondary_calculations | length > 0 %} 37 | {%- do exceptions.raise_compiler_error("Using secondary calculations without a grain is not supported.") %} 38 | {% endif %} 39 | 40 | {% for metric_name in metric_tree.full_set %} 41 | {% if metrics_dictionary[metric_name].window is not none%} 42 | {%- do exceptions.raise_compiler_error("Aggregating without a grain does not support metrics with window definitions.") %} 43 | {% endif%} 44 | {% endfor%} 45 | 46 | {% endif %} 47 | 48 | {% endmacro %} -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.yml: -------------------------------------------------------------------------------- 1 | name: ✨ Feature 2 | description: Propose a straightforward extension of dbt_metrics functionality 3 | title: "[Feature] " 4 | labels: ["enhancement", "triage"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to fill out this feature request! 10 | - type: checkboxes 11 | attributes: 12 | label: Is this your first time submitting a feature request? 13 | description: > 14 | We want to make sure that features are distinct and discoverable, 15 | so that other members of the community can find them and offer their thoughts. 16 | 17 | Issues are the right place to request straightforward extensions of existing dbt_metrics functionality. 18 | For "big ideas" about future capabilities of dbt_metrics, we ask that you open a 19 | [discussion](https://github.com/dbt-labs/dbt-core/discussions) in the "Ideas" category instead. 20 | options: 21 | - label: I have read the [expectations for open source contributors](https://docs.getdbt.com/docs/contributing/oss-expectations) 22 | required: true 23 | - label: I have searched the existing issues, and I could not find an existing issue for this feature 24 | required: true 25 | - label: I am requesting a straightforward extension of existing dbt functionality, rather than a Big Idea better suited to a discussion 26 | required: true 27 | - type: textarea 28 | attributes: 29 | label: Describe the feature 30 | description: A clear and concise description of what you want to happen. 31 | validations: 32 | required: true 33 | - type: textarea 34 | attributes: 35 | label: Describe alternatives you've considered 36 | description: | 37 | A clear and concise description of any alternative solutions or features you've considered. 38 | validations: 39 | required: false 40 | - type: textarea 41 | attributes: 42 | label: Who will this benefit? 43 | description: | 44 | What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly. 45 | validations: 46 | required: false 47 | - type: input 48 | attributes: 49 | label: Are you interested in contributing this feature? 50 | description: Let us know if you want to write some code, and how we can help. 51 | validations: 52 | required: false 53 | - type: textarea 54 | attributes: 55 | label: Anything else? 56 | description: | 57 | Links? References? Anything that will give us more context about the feature you are suggesting! 58 | validations: 59 | required: false -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_no_time_grain_secondary_calc.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | custom_calendar_sql 12 | ) 13 | 14 | # models/no_time_grain_base_sum_metric.sql 15 | no_time_grain_base_sum_metric_sql = """ 16 | select * 17 | from 18 | {{ metrics.calculate( 19 | metric('no_time_grain_base_sum_metric'), 20 | secondary_calculations=[ 21 | metrics.period_over_period(comparison_strategy="difference", interval=1, alias = "1mth") 22 | ] 23 | ) 24 | }} 25 | """ 26 | 27 | # models/no_time_grain_base_sum_metric.yml 28 | no_time_grain_base_sum_metric_yml = """ 29 | version: 2 30 | metrics: 31 | - name: no_time_grain_base_sum_metric 32 | model: ref('fact_orders') 33 | label: Total Discount ($) 34 | calculation_method: sum 35 | expression: order_total 36 | dimensions: 37 | - had_discount 38 | - order_country 39 | """ 40 | 41 | class TestNoTimeGrainSecondaryCalcMetric: 42 | 43 | # configuration in dbt_project.yml 44 | @pytest.fixture(scope="class") 45 | def project_config_update(self): 46 | return { 47 | "name": "example", 48 | "models": {"+materialized": "table"}, 49 | "vars":{ 50 | "dbt_metrics_calendar_model": "custom_calendar", 51 | "custom_calendar_dimension_list": ["is_weekend"] 52 | } 53 | } 54 | 55 | # install current repo as package 56 | @pytest.fixture(scope="class") 57 | def packages(self): 58 | return { 59 | "packages": [ 60 | {"local": os.getcwd()} 61 | ] 62 | } 63 | 64 | # everything that goes in the "seeds" directory 65 | @pytest.fixture(scope="class") 66 | def seeds(self): 67 | return { 68 | "fact_orders_source.csv": fact_orders_source_csv, 69 | } 70 | 71 | # everything that goes in the "models" directory 72 | @pytest.fixture(scope="class") 73 | def models(self): 74 | return { 75 | "fact_orders.sql": fact_orders_sql, 76 | "fact_orders.yml": fact_orders_yml, 77 | "custom_calendar.sql": custom_calendar_sql, 78 | "no_time_grain_base_sum_metric.sql": no_time_grain_base_sum_metric_sql, 79 | "no_time_grain_base_sum_metric.yml": no_time_grain_base_sum_metric_yml 80 | } 81 | 82 | def test_invalid_no_time_grain_secondary_calc(self,project,): 83 | # initial run 84 | run_dbt(["deps"]) 85 | run_dbt(["seed"]) 86 | run_dbt(["run"],expect_pass = False) 87 | 88 | -------------------------------------------------------------------------------- /macros/sql_gen/gen_primary_metric_aggregate.sql: -------------------------------------------------------------------------------- 1 | 2 | {%- macro gen_primary_metric_aggregate(aggregate, expression) -%} 3 | {{ return(adapter.dispatch('gen_primary_metric_aggregate', 'metrics')(aggregate, expression)) }} 4 | {%- endmacro -%} 5 | 6 | {%- macro default__gen_primary_metric_aggregate(aggregate, expression) -%} 7 | 8 | {%- if aggregate == 'count' -%} 9 | {{ return(adapter.dispatch('metric_count', 'metrics')(expression)) }} 10 | 11 | {%- elif aggregate == 'count_distinct' -%} 12 | {{ return(adapter.dispatch('metric_count_distinct', 'metrics')(expression)) }} 13 | 14 | {%- elif aggregate == 'average' -%} 15 | {{ return(adapter.dispatch('metric_average', 'metrics')(expression)) }} 16 | 17 | {%- elif aggregate == 'max' -%} 18 | {{ return(adapter.dispatch('metric_max', 'metrics')(expression)) }} 19 | 20 | {%- elif aggregate == 'min' -%} 21 | {{ return(adapter.dispatch('metric_min', 'metrics')(expression)) }} 22 | 23 | {%- elif aggregate == 'sum' -%} 24 | {{ return(adapter.dispatch('metric_sum', 'metrics')(expression)) }} 25 | 26 | {%- elif aggregate == 'median' -%} 27 | {{ return(adapter.dispatch('metric_median', 'metrics')(expression)) }} 28 | 29 | {%- elif aggregate == 'derived' -%} 30 | {{ return(adapter.dispatch('metric_derived', 'metrics')(expression)) }} 31 | 32 | {%- else -%} 33 | {%- do exceptions.raise_compiler_error("Unknown aggregation style: " ~ aggregate) -%} 34 | {%- endif -%} 35 | {%- endmacro -%} 36 | 37 | {% macro default__metric_count(expression) %} 38 | count({{ expression }}) 39 | {%- endmacro -%} 40 | 41 | {% macro default__metric_count_distinct(expression) %} 42 | count(distinct {{ expression }}) 43 | {%- endmacro -%} 44 | 45 | {% macro default__metric_average(expression) %} 46 | avg({{ expression }}) 47 | {%- endmacro -%} 48 | 49 | {% macro redshift__metric_average(expression) %} 50 | avg(cast({{ expression }} as float)) 51 | {%- endmacro -%} 52 | 53 | {% macro default__metric_max(expression) %} 54 | max({{ expression }}) 55 | {%- endmacro -%} 56 | 57 | {% macro default__metric_min(expression) %} 58 | min({{ expression }}) 59 | {%- endmacro -%} 60 | 61 | {% macro default__metric_sum(expression) %} 62 | sum({{ expression }}) 63 | {%- endmacro -%} 64 | 65 | {% macro default__metric_median(expression) %} 66 | median({{ expression }}) 67 | {%- endmacro -%} 68 | 69 | {% macro bigquery__metric_median(expression) %} 70 | any_value({{ expression }}) 71 | {%- endmacro -%} 72 | 73 | {% macro postgres__metric_median(expression) %} 74 | percentile_cont(0.5) within group (order by {{ expression }}) 75 | {%- endmacro -%} 76 | 77 | {% macro default__metric_derived(expression) %} 78 | {{ expression }} 79 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/secondary_calculations/generate_secondary_calculation_alias.sql: -------------------------------------------------------------------------------- 1 | {% macro generate_secondary_calculation_alias(metric_name, calc_config, grain, is_multiple_metrics) %} 2 | 3 | {{ return(adapter.dispatch('generate_secondary_calculation_alias', 'metrics')(metric_name, calc_config, grain, is_multiple_metrics)) }} 4 | 5 | {% endmacro %} 6 | 7 | {% macro default__generate_secondary_calculation_alias(metric_name, calc_config, grain, is_multiple_metrics) %} 8 | {%- if calc_config.alias -%} 9 | {%- if is_multiple_metrics -%} 10 | {%- do return(metric_name ~ "_" ~ calc_config.alias) -%} 11 | {%- else -%} 12 | {% do return(calc_config.alias) %} 13 | {%- endif -%} 14 | {%- endif -%} 15 | 16 | {%- set calc_type = calc_config.calculation -%} 17 | {%- if calc_type == 'period_over_period' -%} 18 | {%- if is_multiple_metrics -%} 19 | {%- do return(metric_name ~ "_" ~ calc_config.comparison_strategy ~ "_to_" ~ calc_config.interval ~ "_" ~ grain ~ "_ago") %} 20 | {%- else -%} 21 | {%- do return(calc_config.comparison_strategy ~ "_to_" ~ calc_config.interval ~ "_" ~ grain ~ "_ago") %} 22 | {%- endif -%} 23 | 24 | {%- elif calc_type == 'rolling' %} 25 | {%- if is_multiple_metrics -%} 26 | {%- if calc_config.interval -%} 27 | {%- do return(metric_name ~ "_" ~ "rolling_" ~ calc_config.aggregate ~ "_" ~ calc_config.interval ~ "_" ~ grain) %} 28 | {%- else -%} 29 | {%- do return(metric_name ~ "_" ~ "rolling_" ~ calc_config.aggregate) %} 30 | {%- endif -%} 31 | {%- else -%} 32 | {%- if calc_config.interval -%} 33 | {%- do return("rolling_" ~ calc_config.aggregate ~ "_" ~ calc_config.interval ~ "_" ~ grain) %} 34 | {%- else -%} 35 | {%- do return("rolling_" ~ calc_config.aggregate) %} 36 | {%- endif -%} 37 | {%- endif -%} 38 | 39 | {%- elif calc_type == 'period_to_date' %} 40 | {% if is_multiple_metrics %} 41 | {%- do return(metric_name ~ "_" ~ calc_config.aggregate ~ "_for_" ~ calc_config.period) %} 42 | {% else %} 43 | {%- do return(calc_config.aggregate ~ "_for_" ~ calc_config.period) %} 44 | {% endif %} 45 | 46 | {%- elif calc_type == 'prior' %} 47 | {% if is_multiple_metrics %} 48 | {%- do return(metric_name ~ "_" ~ calc_config.interval ~ "_" ~ grain ~ "s_prior") %} 49 | {% else %} 50 | {%- do return(calc_config.interval ~ "_" ~ grain ~ "s_prior") %} 51 | {% endif %} 52 | 53 | {%- else %} 54 | {%- do exceptions.raise_compiler_error("Can't generate alias for unknown secondary calculation: " ~ calc_type ~ ". calc_config: " ~ calc_config) %} 55 | {%- endif %} 56 | 57 | {{ calc_sql }} 58 | {% endmacro %} -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_backwards_compatability_metric_list.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/backwards_compatability_metric.sql 14 | backwards_compatability_metric_sql = """ 15 | select * 16 | from 17 | {{ metrics.metric( 18 | metric_name=['backwards_compatability_metric','base_average_metric'], 19 | grain='month' 20 | ) 21 | }} 22 | """ 23 | 24 | # models/backwards_compatability_metric.yml 25 | backwards_compatability_metric_yml = """ 26 | version: 2 27 | models: 28 | - name: backwards_compatability_metric 29 | 30 | metrics: 31 | - name: backwards_compatability_metric 32 | model: ref('fact_orders') 33 | label: Total Discount ($) 34 | timestamp: order_date 35 | time_grains: [day, week, month] 36 | calculation_method: average 37 | expression: discount_total 38 | dimensions: 39 | - had_discount 40 | - order_country 41 | 42 | - name: base_average_metric 43 | model: ref('fact_orders') 44 | label: Total Discount ($) 45 | timestamp: order_date 46 | time_grains: [day, week, month] 47 | calculation_method: average 48 | expression: discount_total 49 | dimensions: 50 | - had_discount 51 | - order_country 52 | """ 53 | 54 | class TestBackwardsCompatibility: 55 | # configuration in dbt_project.yml 56 | @pytest.fixture(scope="class") 57 | def project_config_update(self): 58 | return { 59 | "name": "example", 60 | "models": {"+materialized": "table"} 61 | } 62 | 63 | # install current repo as package 64 | @pytest.fixture(scope="class") 65 | def packages(self): 66 | return { 67 | "packages": [ 68 | {"local": os.getcwd()} 69 | ] 70 | } 71 | 72 | 73 | # everything that goes in the "seeds" directory 74 | @pytest.fixture(scope="class") 75 | def seeds(self): 76 | return { 77 | "fact_orders_source.csv": fact_orders_source_csv 78 | } 79 | 80 | # everything that goes in the "models" directory 81 | @pytest.fixture(scope="class") 82 | def models(self): 83 | return { 84 | "fact_orders.sql": fact_orders_sql, 85 | "fact_orders.yml": fact_orders_yml, 86 | "backwards_compatability_metric.sql": backwards_compatability_metric_sql, 87 | "backwards_compatability_metric.yml": backwards_compatability_metric_yml 88 | } 89 | 90 | def test_build_completion(self,project,): 91 | results = run_dbt(["deps"]) 92 | results = run_dbt(["seed"]) 93 | 94 | # initial run 95 | results = run_dbt(["run"],expect_pass = False) 96 | -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_period_to_date_average.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/invalid_period_to_date_average.sql 14 | invalid_period_to_date_average_sql = """ 15 | select * 16 | from 17 | {{ metrics.calculate(metric('invalid_period_to_date_average'), 18 | grain='month', 19 | secondary_calculations=[ 20 | metrics.period_to_date(aggregate="average", period="year", alias="this_year_average") 21 | ] 22 | ) 23 | }} 24 | """ 25 | 26 | # models/invalid_period_to_date_average.yml 27 | invalid_period_to_date_average_yml = """ 28 | version: 2 29 | models: 30 | - name: invalid_period_to_date_average 31 | tests: 32 | - metrics.metric_equality: 33 | compare_model: ref('invalid_period_to_date_average__expected') 34 | metrics: 35 | - name: invalid_period_to_date_average 36 | model: ref('fact_orders') 37 | label: Total Discount ($) 38 | timestamp: order_date 39 | time_grains: [day, week, month] 40 | calculation_method: average 41 | expression: discount_total 42 | dimensions: 43 | - had_discount 44 | - order_country 45 | """ 46 | 47 | class TestInvalidPeriodToDateAverage: 48 | 49 | # configuration in dbt_project.yml 50 | @pytest.fixture(scope="class") 51 | def project_config_update(self): 52 | return { 53 | "name": "example", 54 | "models": {"+materialized": "table"} 55 | } 56 | 57 | # install current repo as package 58 | @pytest.fixture(scope="class") 59 | def packages(self): 60 | return { 61 | "packages": [ 62 | {"local": os.getcwd()} 63 | ] 64 | } 65 | 66 | 67 | # everything that goes in the "seeds" directory 68 | @pytest.fixture(scope="class") 69 | def seeds(self): 70 | return { 71 | "fact_orders_source.csv": fact_orders_source_csv 72 | } 73 | 74 | # everything that goes in the "models" directory 75 | @pytest.fixture(scope="class") 76 | def models(self): 77 | return { 78 | "fact_orders.sql": fact_orders_sql, 79 | "fact_orders.yml": fact_orders_yml, 80 | "invalid_period_to_date_average.sql": invalid_period_to_date_average_sql, 81 | "invalid_period_to_date_average.yml": invalid_period_to_date_average_yml 82 | } 83 | 84 | def test_build_completion(self,project,): 85 | # running deps to install package 86 | results = run_dbt(["deps"]) 87 | 88 | # seed seeds 89 | results = run_dbt(["seed"]) 90 | assert len(results) == 1 91 | 92 | # initial run 93 | results = run_dbt(["run"], expect_pass = False) 94 | -------------------------------------------------------------------------------- /macros/variables/get_model_group.sql: -------------------------------------------------------------------------------- 1 | {%- macro get_model_group(models_grouping, metric_model, metric_model_name, metric_name, metric_timestamp=none, metric_filters=none, metric_window=none) -%} 2 | 3 | {#- 4 | This macro is called from get_models_grouping in order to calculate 5 | the group for each model based on the inputs. This allows us to reduce 6 | the complexity of the aforementioned macro because there is a factorial 7 | combination of possibilities based on the inputs, minus some combinations 8 | that are invalid. 9 | 10 | By factorial, we mean that the three potential inputs can be combined in 11 | a multitude of different ways in order to calculate the group. The potential 12 | combinations are: 13 | - timestamp 14 | - filters 15 | - timestamp + window 16 | - timestamp + filters 17 | - timestamp + filters + window 18 | -#} 19 | 20 | {% set metric_model_list = [metric_model_name] %} 21 | 22 | {% if metric_timestamp %} 23 | {% set timestamp_list = [ 24 | metric_timestamp | lower 25 | ]%} 26 | {% else %} 27 | {% set timestamp_list = [] %} 28 | {% endif %} 29 | 30 | {% if metric_window %} 31 | {% set window_list = [ 32 | metric_window.count | lower 33 | ,metric_window.period | lower 34 | ]%} 35 | {% else %} 36 | {% set window_list = [] %} 37 | {% endif %} 38 | 39 | {% if metric_filters %} 40 | {% set filter_list = [] %} 41 | {% for filter in metric_filters %} 42 | {% do filter_list.append(filter.field | lower)%} 43 | {% do filter_list.append(filter.operator | lower)%} 44 | {% do filter_list.append(filter.value | lower)%} 45 | {% endfor %} 46 | {% else %} 47 | {% set filter_list = [] %} 48 | {% endif %} 49 | 50 | {% set group_list = (metric_model_list + timestamp_list + window_list + filter_list) | sort %} 51 | {% set group_name = 'model_' ~ local_md5(group_list | join('_')) %} 52 | 53 | {% if not models_grouping[group_name] %} 54 | {% do models_grouping.update({group_name:{}})%} 55 | {% do models_grouping[group_name].update({'metric_names':{}})%} 56 | {% do models_grouping[group_name].update({'metric_model':metric_model})%} 57 | {% do models_grouping[group_name].update({'timestamp':metric_timestamp})%} 58 | {% do models_grouping[group_name].update({'filters':metric_filters})%} 59 | {% do models_grouping[group_name].update({'window':metric_window})%} 60 | {% do models_grouping[group_name].update({'metric_names':[metric_name]})%} 61 | {% else %} 62 | {% set metric_names = models_grouping[group_name]['metric_names'] %} 63 | {% do metric_names.append(metric_name)%} 64 | {% do models_grouping[group_name].update({'metric_names':metric_names})%} 65 | {% endif %} 66 | 67 | {% do return(metrics_grouping) %} 68 | 69 | {%- endmacro -%} -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_backwards_compatability_expression_metric.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/backwards_compatability_metric.sql 14 | backwards_compatability_derived_metric_sql = """ 15 | select * 16 | from 17 | {{ metrics.metric( 18 | metric_name='backwards_compatability_derived_metric', 19 | grain='month' 20 | ) 21 | }} 22 | """ 23 | 24 | # models/backwards_compatability_metric.yml 25 | backwards_compatability_metric_yml = """ 26 | version: 2 27 | models: 28 | - name: backwards_compatability_derived_metric 29 | 30 | metrics: 31 | - name: backwards_compatability_metric 32 | model: ref('fact_orders') 33 | label: Total Discount ($) 34 | timestamp: order_date 35 | time_grains: [day, week, month] 36 | calculation_method: average 37 | expression: discount_total 38 | dimensions: 39 | - had_discount 40 | - order_country 41 | 42 | - name: backwards_compatability_derived_metric 43 | label: Total Discount ($) 44 | timestamp: order_date 45 | time_grains: [day, week, month] 46 | calculation_method: derived 47 | expression: "{{metric('backwards_compatability_metric')}} + 1" 48 | dimensions: 49 | - had_discount 50 | - order_country 51 | """ 52 | 53 | class TestBackwardsCompatibilityDerivedMetric: 54 | # configuration in dbt_project.yml 55 | @pytest.fixture(scope="class") 56 | def project_config_update(self): 57 | return { 58 | "name": "example", 59 | "models": {"+materialized": "table"} 60 | } 61 | 62 | # install current repo as package 63 | @pytest.fixture(scope="class") 64 | def packages(self): 65 | return { 66 | "packages": [ 67 | {"local": os.getcwd()} 68 | ] 69 | } 70 | 71 | 72 | # everything that goes in the "seeds" directory 73 | @pytest.fixture(scope="class") 74 | def seeds(self): 75 | return { 76 | "fact_orders_source.csv": fact_orders_source_csv 77 | } 78 | 79 | # everything that goes in the "models" directory 80 | @pytest.fixture(scope="class") 81 | def models(self): 82 | return { 83 | "fact_orders.sql": fact_orders_sql, 84 | "fact_orders.yml": fact_orders_yml, 85 | "backwards_compatability_derived_metric.sql": backwards_compatability_derived_metric_sql, 86 | "backwards_compatability_metric.yml": backwards_compatability_metric_yml 87 | } 88 | 89 | def test_build_completion(self,project,): 90 | results = run_dbt(["deps"]) 91 | results = run_dbt(["seed"]) 92 | 93 | # initial run 94 | results = run_dbt(["run"],expect_pass = False) 95 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | 4 | # Import the standard functional fixtures as a plugin 5 | # Note: fixtures with session scope need to be local 6 | pytest_plugins = ["dbt.tests.fixtures.project"] 7 | 8 | # The profile dictionary, used to write out profiles.yml 9 | # dbt will supply a unique schema per test, so we do not specify 'schema' here 10 | 11 | # We use os.environ here instead of os.getenv because environ with [] input will 12 | # return a KeyError exception instead of None or Default Value. It's better to know 13 | # when the error is from the environment variables and not have it potentially lead 14 | # you down a red herring path with other issues. 15 | @pytest.fixture(scope="class") 16 | def dbt_profile_target(): 17 | 18 | if os.environ['dbt_target'] == 'postgres': 19 | return { 20 | 'type': 'postgres', 21 | 'threads': 8, 22 | 'host': os.environ['POSTGRES_TEST_HOST'], 23 | 'user': os.environ['POSTGRES_TEST_USER'], 24 | 'password': os.environ['POSTGRES_TEST_PASSWORD'], 25 | 'port': int(os.environ['POSTGRES_TEST_PORT']), 26 | 'database': os.environ['POSTGRES_TEST_DB'], 27 | } 28 | 29 | if os.environ['dbt_target'] == 'redshift': 30 | return { 31 | 'type': 'redshift', 32 | 'threads': 8, 33 | 'host': os.environ['REDSHIFT_TEST_HOST'], 34 | 'user': os.environ['REDSHIFT_TEST_USER'], 35 | 'pass': os.environ['REDSHIFT_TEST_PASS'], 36 | 'dbname': os.environ['REDSHIFT_TEST_DBNAME'], 37 | 'port': int(os.environ['REDSHIFT_TEST_PORT']), 38 | } 39 | 40 | if os.environ['dbt_target'] == 'snowflake': 41 | return { 42 | 'type': 'snowflake', 43 | 'threads': 8, 44 | 'account': os.environ['SNOWFLAKE_TEST_ACCOUNT'], 45 | 'user': os.environ['SNOWFLAKE_TEST_USER'], 46 | 'password': os.environ['SNOWFLAKE_TEST_PASSWORD'], 47 | 'role': os.environ['SNOWFLAKE_TEST_ROLE'], 48 | 'database': os.environ['SNOWFLAKE_TEST_DATABASE'], 49 | 'warehouse': os.environ['SNOWFLAKE_TEST_WAREHOUSE'], 50 | } 51 | 52 | if os.environ['dbt_target'] == 'bigquery': 53 | return { 54 | 'type': 'bigquery', 55 | 'threads': 8, 56 | 'method': 'service-account', 57 | 'project': os.environ['BIGQUERY_TEST_PROJECT'], 58 | 'keyfile': os.environ['BIGQUERY_SERVICE_KEY_PATH'], 59 | } 60 | 61 | if os.environ['dbt_target'] == 'databricks': 62 | return { 63 | 'type': 'databricks', 64 | 'threads': 8, 65 | 'catalog': os.environ['DATABRICKS_TEST_CATALOG'], 66 | 'schema': os.environ['DATABRICKS_TEST_SCHEMA'], 67 | 'host': os.environ['DATABRICKS_TEST_HOST'], 68 | 'http_path': os.environ['DATABRICKS_TEST_HTTP_PATH'], 69 | 'token': os.environ['DATABRICKS_TEST_TOKEN'], 70 | } -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_ephemeral_model.py: -------------------------------------------------------------------------------- 1 | from configparser import ParsingError 2 | from struct import pack 3 | import os 4 | import pytest 5 | from dbt.tests.util import run_dbt 6 | 7 | # our file contents 8 | from tests.functional.fixtures import ( 9 | fact_orders_source_csv, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/fact_orders.sql 14 | fact_orders_sql = """ 15 | 16 | {{ config(materialized='ephemeral') }} 17 | 18 | select 19 | * 20 | ,round(order_total - (order_total/2)) as discount_total 21 | from {{ref('fact_orders_source')}} 22 | """ 23 | 24 | # models/invalid_ephemeral_model.sql 25 | invalid_ephemeral_model_sql = """ 26 | select * 27 | from 28 | {{ metrics.calculate(metric('invalid_ephemeral_model'), 29 | grain='month' 30 | ) 31 | }} 32 | """ 33 | 34 | # models/invalid_ephemeral_model.yml 35 | invalid_ephemeral_model_yml = """ 36 | version: 2 37 | models: 38 | - name: invalid_ephemeral_model 39 | 40 | metrics: 41 | - name: invalid_ephemeral_model 42 | model: ref('fact_orders') 43 | label: Total Discount ($) 44 | timestamp: order_date 45 | time_grains: [day, week, month] 46 | calculation_method: count 47 | expression: order_total 48 | dimensions: 49 | - had_discount 50 | - order_country 51 | config: 52 | treat_null_values_as_zero: banana 53 | """ 54 | 55 | class TestInvalidMetricConfig: 56 | 57 | # configuration in dbt_project.yml 58 | @pytest.fixture(scope="class") 59 | def project_config_update(self): 60 | return { 61 | "name": "example", 62 | "models": {"+materialized": "table"} 63 | } 64 | 65 | # install current repo as package 66 | @pytest.fixture(scope="class") 67 | def packages(self): 68 | return { 69 | "packages": [ 70 | {"local": os.getcwd()} 71 | ] 72 | } 73 | 74 | 75 | # everything that goes in the "seeds" directory 76 | @pytest.fixture(scope="class") 77 | def seeds(self): 78 | return { 79 | "fact_orders_source.csv": fact_orders_source_csv 80 | } 81 | 82 | # everything that goes in the "models" directory 83 | @pytest.fixture(scope="class") 84 | def models(self): 85 | return { 86 | "fact_orders.sql": fact_orders_sql, 87 | "fact_orders.yml": fact_orders_yml, 88 | "invalid_ephemeral_model.sql": invalid_ephemeral_model_sql, 89 | "invalid_ephemeral_model.yml": invalid_ephemeral_model_yml 90 | } 91 | 92 | def test_metric_config_value(self,project,): 93 | # initial run 94 | results = run_dbt(["deps"]) 95 | 96 | # seed seeds 97 | results = run_dbt(["seed"]) 98 | assert len(results) == 1 99 | 100 | # Here we expect the run to fail because the value provided 101 | # in the where clause isn't included in the final dataset 102 | run_dbt(["run"], expect_pass = False) -------------------------------------------------------------------------------- /macros/sql_gen/gen_property_to_aggregate.sql: -------------------------------------------------------------------------------- 1 | {%- macro gen_property_to_aggregate(metric_dictionary, grain, dimensions, calendar_dimensions) -%} 2 | {{ return(adapter.dispatch('gen_property_to_aggregate', 'metrics')(metric_dictionary, grain, dimensions, calendar_dimensions)) }} 3 | {%- endmacro -%} 4 | 5 | {% macro default__gen_property_to_aggregate(metric_dictionary, grain, dimensions, calendar_dimensions) %} 6 | {% if metric_dictionary.calculation_method == 'median' -%} 7 | {{ return(adapter.dispatch('property_to_aggregate_median', 'metrics')(metric_dictionary, grain, dimensions, calendar_dimensions)) }} 8 | 9 | {% elif metric_dictionary.calculation_method == 'count' -%} 10 | {{ return(adapter.dispatch('property_to_aggregate_count', 'metrics')(metric_dictionary)) }} 11 | 12 | {% elif metric_dictionary.expression and metric_dictionary.expression | replace('*', '') | trim != '' %} 13 | {{ return(adapter.dispatch('property_to_aggregate_default', 'metrics')(metric_dictionary)) }} 14 | 15 | {% else %} 16 | {%- do exceptions.raise_compiler_error("Expression to aggregate is required for non-count aggregation in metric `" ~ metric_dictionary.name ~ "`") -%} 17 | {% endif %} 18 | 19 | {%- endmacro -%} 20 | 21 | {% macro default__property_to_aggregate_median(metric_dictionary, grain, dimensions, calendar_dimensions) %} 22 | ({{metric_dictionary.expression }}) as property_to_aggregate__{{metric_dictionary.name}} 23 | {%- endmacro -%} 24 | 25 | {% macro bigquery__property_to_aggregate_median(metric_dictionary, grain, dimensions, calendar_dimensions) %} 26 | 27 | percentile_cont({{metric_dictionary.expression }}, 0.5) over ( 28 | {% if grain or dimensions | length > 0 or calendar_dimensions | length > 0 -%} 29 | partition by 30 | {% if grain -%} 31 | calendar.date_{{ grain }} 32 | {%- endif %} 33 | {% for dim in dimensions -%} 34 | {%- if loop.first and not grain-%} 35 | base_model.{{ dim }} 36 | {%- else -%} 37 | ,base_model.{{ dim }} 38 | {%- endif -%} 39 | {%- endfor -%} 40 | {% for calendar_dim in calendar_dimensions -%} 41 | {%- if loop.first and dimensions | length == 0 and not grain %} 42 | calendar.{{ calendar_dim }} 43 | {%else -%} 44 | ,calendar.{{ calendar_dim }} 45 | {%- endif -%} 46 | {%- endfor %} 47 | {%- endif %} 48 | ) as property_to_aggregate__{{metric_dictionary.name}} 49 | 50 | {%- endmacro -%} 51 | 52 | {% macro default__property_to_aggregate_count(metric_dictionary) %} 53 | 1 as property_to_aggregate__{{metric_dictionary.name}} 54 | {%- endmacro -%} 55 | 56 | {% macro default__property_to_aggregate_default(metric_dictionary) %} 57 | ({{metric_dictionary.expression }}) as property_to_aggregate__{{metric_dictionary.name}} 58 | {%- endmacro -%} -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_derived_metric_filter.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/invalid_derived_metric.sql 14 | invalid_derived_metric_sql = """ 15 | select * 16 | from 17 | {{ metrics.calculate(metric('invalid_derived_metric'), 18 | grain='month' 19 | ) 20 | }} 21 | """ 22 | 23 | # models/invalid_derived_metric.yml 24 | invalid_derived_metric_yml = """ 25 | version: 2 26 | models: 27 | - name: invalid_derived_metric 28 | 29 | metrics: 30 | - name: base_sum_metric 31 | model: ref('fact_orders') 32 | label: Order Total ($) 33 | timestamp: order_date 34 | time_grains: [day, week, month] 35 | calculation_method: sum 36 | expression: order_total 37 | dimensions: 38 | - had_discount 39 | - order_country 40 | 41 | - name: invalid_derived_metric 42 | label: Total Discount ($) 43 | timestamp: order_date 44 | time_grains: [day, week, month] 45 | calculation_method: derived 46 | expression: "{{metric('base_sum_metric')}} - 1" 47 | dimensions: 48 | - had_discount 49 | - order_country 50 | 51 | filters: 52 | - field: had_discount 53 | operator: 'is' 54 | value: 'true' 55 | """ 56 | 57 | class TestInvalidDerivedMetric: 58 | 59 | # configuration in dbt_project.yml 60 | @pytest.fixture(scope="class") 61 | def project_config_update(self): 62 | return { 63 | "name": "example", 64 | "models": {"+materialized": "table"} 65 | } 66 | 67 | # install current repo as package 68 | @pytest.fixture(scope="class") 69 | def packages(self): 70 | return { 71 | "packages": [ 72 | {"local": os.getcwd()} 73 | ] 74 | } 75 | 76 | 77 | # everything that goes in the "seeds" directory 78 | @pytest.fixture(scope="class") 79 | def seeds(self): 80 | return { 81 | "fact_orders_source.csv": fact_orders_source_csv 82 | } 83 | 84 | # everything that goes in the "models" directory 85 | @pytest.fixture(scope="class") 86 | def models(self): 87 | return { 88 | "fact_orders.sql": fact_orders_sql, 89 | "fact_orders.yml": fact_orders_yml, 90 | "invalid_derived_metric.sql": invalid_derived_metric_sql, 91 | "invalid_derived_metric.yml": invalid_derived_metric_yml 92 | } 93 | 94 | def test_invalid_derived_metric(self,project,): 95 | # running deps to install package 96 | results = run_dbt(["deps"]) 97 | 98 | # seed seeds 99 | results = run_dbt(["seed"]) 100 | assert len(results) == 1 101 | 102 | # Here we expect the run to fail because the incorrect 103 | # config won't allow it to compile 104 | run_dbt(["run"], expect_pass = False) -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_no_time_grain_calendar_dimension.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | from dbt.tests.util import run_dbt 6 | 7 | # our file contents 8 | from tests.functional.fixtures import ( 9 | fact_orders_source_csv, 10 | fact_orders_sql, 11 | fact_orders_yml, 12 | custom_calendar_sql 13 | ) 14 | 15 | # models/no_timestamp_base_sum_metric.sql 16 | no_timestamp_base_sum_metric_sql = """ 17 | select * 18 | from 19 | {{ metrics.calculate(metric('no_timestamp_base_sum_metric'), 20 | dimensions=["is_weekend"] 21 | ) 22 | }} 23 | """ 24 | 25 | # models/no_timestamp_base_sum_metric.yml 26 | no_timestamp_base_sum_metric_yml = """ 27 | version: 2 28 | models: 29 | - name: no_timestamp_base_sum_metric 30 | tests: 31 | - metrics.metric_equality: 32 | compare_model: ref('no_timestamp_base_sum_metric__expected') 33 | metrics: 34 | - name: no_timestamp_base_sum_metric 35 | model: ref('fact_orders') 36 | label: Total Discount ($) 37 | calculation_method: sum 38 | expression: order_total 39 | dimensions: 40 | - had_discount 41 | - order_country 42 | """ 43 | 44 | # seeds/base_sum_metric__expected.csv 45 | no_timestamp_base_sum_metric__expected_csv = """ 46 | is_weekend,no_timestamp_base_sum_metric 47 | true,14 48 | """.lstrip() 49 | 50 | class TestNoTimestampCustomCalendarDimensionsMetric: 51 | 52 | # configuration in dbt_project.yml 53 | @pytest.fixture(scope="class") 54 | def project_config_update(self): 55 | return { 56 | "name": "example", 57 | "models": {"+materialized": "table"}, 58 | "vars":{ 59 | "dbt_metrics_calendar_model": "custom_calendar", 60 | "custom_calendar_dimension_list": ["is_weekend"] 61 | } 62 | } 63 | 64 | # install current repo as package 65 | @pytest.fixture(scope="class") 66 | def packages(self): 67 | return { 68 | "packages": [ 69 | {"local": os.getcwd()} 70 | ] 71 | } 72 | 73 | # everything that goes in the "seeds" directory 74 | @pytest.fixture(scope="class") 75 | def seeds(self): 76 | return { 77 | "fact_orders_source.csv": fact_orders_source_csv, 78 | "no_timestamp_base_sum_metric__expected.csv": no_timestamp_base_sum_metric__expected_csv, 79 | } 80 | 81 | # everything that goes in the "models" directory 82 | @pytest.fixture(scope="class") 83 | def models(self): 84 | return { 85 | "fact_orders.sql": fact_orders_sql, 86 | "fact_orders.yml": fact_orders_yml, 87 | "custom_calendar.sql": custom_calendar_sql, 88 | "no_timestamp_base_sum_metric.sql": no_timestamp_base_sum_metric_sql, 89 | "no_timestamp_base_sum_metric.yml": no_timestamp_base_sum_metric_yml 90 | } 91 | 92 | def test_invalid_no_time_grain_calendar_dimension(self,project): 93 | # initial run 94 | run_dbt(["deps"]) 95 | run_dbt(["seed"]) 96 | run_dbt(["run"],expect_pass = False) 97 | -------------------------------------------------------------------------------- /tests/functional/metric_options/old_metric_spec/test_old_metric_spec.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/old_spec_metric.sql 14 | old_spec_metric_sql = """ 15 | select * 16 | from 17 | {{ metrics.calculate(metric('old_spec_metric'), 18 | grain='month' 19 | ) 20 | }} 21 | """ 22 | 23 | # models/old_spec_metric.yml 24 | old_spec_metric_yml = """ 25 | version: 2 26 | models: 27 | - name: old_spec_metric 28 | tests: 29 | - metrics.metric_equality: 30 | compare_model: ref('old_spec_metric__expected') 31 | metrics: 32 | - name: old_spec_metric 33 | model: ref('fact_orders') 34 | label: Total Discount ($) 35 | timestamp: order_date 36 | time_grains: [day, week, month] 37 | type: sum 38 | sql: order_total 39 | dimensions: 40 | - had_discount 41 | - order_country 42 | """ 43 | 44 | # seeds/old_spec_metric__expected.csv 45 | old_spec_metric__expected_csv = """ 46 | date_month,old_spec_metric 47 | 2022-01-01,8 48 | 2022-02-01,6 49 | """.lstrip() 50 | 51 | class TestOldSpecMetric: 52 | 53 | # configuration in dbt_project.yml 54 | @pytest.fixture(scope="class") 55 | def project_config_update(self): 56 | return { 57 | "name": "example", 58 | "models": {"+materialized": "table"} 59 | } 60 | 61 | # install current repo as package 62 | @pytest.fixture(scope="class") 63 | def packages(self): 64 | return { 65 | "packages": [ 66 | {"local": os.getcwd()} 67 | ] 68 | } 69 | 70 | 71 | # everything that goes in the "seeds" directory 72 | @pytest.fixture(scope="class") 73 | def seeds(self): 74 | return { 75 | "fact_orders_source.csv": fact_orders_source_csv, 76 | "old_spec_metric__expected.csv": old_spec_metric__expected_csv, 77 | } 78 | 79 | # everything that goes in the "models" directory 80 | @pytest.fixture(scope="class") 81 | def models(self): 82 | return { 83 | "fact_orders.sql": fact_orders_sql, 84 | "fact_orders.yml": fact_orders_yml, 85 | "old_spec_metric.sql": old_spec_metric_sql, 86 | "old_spec_metric.yml": old_spec_metric_yml 87 | } 88 | 89 | def test_build_completion(self,project,): 90 | # running deps to install package 91 | results = run_dbt(["deps"]) 92 | 93 | # seed seeds 94 | results = run_dbt(["seed"]) 95 | assert len(results) == 2 96 | 97 | # initial run 98 | results = run_dbt(["run"]) 99 | assert len(results) == 3 100 | 101 | # test tests 102 | results = run_dbt(["test"]) # expect passing test 103 | assert len(results) == 1 104 | 105 | # # # validate that the results include pass 106 | result_statuses = sorted(r.status for r in results) 107 | assert result_statuses == ["pass"] -------------------------------------------------------------------------------- /macros/misc/metrics__equality.sql: -------------------------------------------------------------------------------- 1 | {% test metric_equality(model, compare_model, compare_columns=none) %} 2 | {{ return(adapter.dispatch('test_metric_equality', 'metrics')(model, compare_model, compare_columns)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_metric_equality(model, compare_model, compare_columns=none) %} 6 | 7 | {% set set_diff %} 8 | count(*) + coalesce(abs( 9 | sum(case when which_diff = 'a_minus_b' then 1 else 0 end) - 10 | sum(case when which_diff = 'b_minus_a' then 1 else 0 end) 11 | ), 0) 12 | {% endset %} 13 | 14 | {#-- Needs to be set at parse time, before we return '' below --#} 15 | {{ config(fail_calc = set_diff) }} 16 | 17 | {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} 18 | {%- if not execute -%} 19 | {{ return('') }} 20 | {% endif %} 21 | 22 | -- setup 23 | {%- do metrics._metric_is_relation(model, 'test_metric_equality') -%} 24 | 25 | {#- 26 | If the compare_cols arg is provided, we can run this test without querying the 27 | information schema — this allows the model to be an ephemeral model 28 | -#} 29 | 30 | {%- if not compare_columns -%} 31 | {%- do metrics._metric_is_ephemeral(model, 'test_metric_equality') -%} 32 | {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%} 33 | {%- endif -%} 34 | 35 | {% set compare_cols_csv = compare_columns | join(', ') %} 36 | 37 | with a as ( 38 | 39 | select * from {{ model }} 40 | 41 | ), 42 | 43 | b as ( 44 | 45 | select * from {{ compare_model }} 46 | 47 | ), 48 | 49 | a_minus_b as ( 50 | 51 | select {{compare_cols_csv}} from a 52 | {{ except() }} 53 | select {{compare_cols_csv}} from b 54 | 55 | ), 56 | 57 | b_minus_a as ( 58 | 59 | select {{compare_cols_csv}} from b 60 | {{ except() }} 61 | select {{compare_cols_csv}} from a 62 | 63 | ), 64 | 65 | unioned as ( 66 | 67 | select 'a_minus_b' as which_diff, a_minus_b.* from a_minus_b 68 | union all 69 | select 'b_minus_a' as which_diff, b_minus_a.* from b_minus_a 70 | 71 | ) 72 | 73 | select * from unioned 74 | 75 | {% endmacro %} 76 | 77 | 78 | {% macro _metric_is_relation(obj, macro) %} 79 | {%- if not (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation')) -%} 80 | {%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} 81 | {%- endif -%} 82 | {% endmacro %} 83 | 84 | {% macro _metric_is_ephemeral(obj, macro) %} 85 | {%- if obj.is_cte -%} 86 | {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} 87 | {% if obj.name.startswith(ephemeral_prefix) %} 88 | {% set model_name = obj.name[(ephemeral_prefix|length):] %} 89 | {% else %} 90 | {% set model_name = obj.name %} 91 | {%- endif -%} 92 | {% set error_message %} 93 | The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema. 94 | 95 | `{{ model_name }}` is an ephemeral model. Consider making it a view or table instead. 96 | {% endset %} 97 | {%- do exceptions.raise_compiler_error(error_message) -%} 98 | {%- endif -%} 99 | {% endmacro %} -------------------------------------------------------------------------------- /tests/functional/invalid_configs/test_invalid_where.py: -------------------------------------------------------------------------------- 1 | from struct import pack 2 | import os 3 | import pytest 4 | from dbt.tests.util import run_dbt 5 | 6 | # our file contents 7 | from tests.functional.fixtures import ( 8 | fact_orders_source_csv, 9 | fact_orders_sql, 10 | fact_orders_yml, 11 | ) 12 | 13 | # models/invalid_where.sql 14 | invalid_where_sql = """ 15 | select * 16 | from 17 | {{ metrics.calculate(metric('invalid_where'), 18 | grain='month', 19 | dimensions=['had_discount'], 20 | where="order_country='Japan'" 21 | ) 22 | }} 23 | """ 24 | 25 | # models/invalid_where.yml 26 | invalid_where_yml = """ 27 | version: 2 28 | models: 29 | - name: invalid_where 30 | tests: 31 | - metrics.metric_equality: 32 | compare_model: ref('invalid_where__expected') 33 | metrics: 34 | - name: invalid_where 35 | model: ref('fact_orders') 36 | label: Total Discount ($) 37 | timestamp: order_date 38 | time_grains: [day, week, month] 39 | calculation_method: sum 40 | expression: order_total 41 | dimensions: 42 | - had_discount 43 | - order_country 44 | """ 45 | 46 | class TestInvalidWhereMetric: 47 | 48 | # configuration in dbt_project.yml 49 | # setting bigquery as table to get around query complexity 50 | # resource constraints with compunding views 51 | if os.getenv('dbt_target') == 'bigquery': 52 | @pytest.fixture(scope="class") 53 | def project_config_update(self): 54 | return { 55 | "name": "example", 56 | "models": {"+materialized": "table"} 57 | } 58 | else: 59 | @pytest.fixture(scope="class") 60 | def project_config_update(self): 61 | return { 62 | "name": "example", 63 | "models": {"+materialized": "view"} 64 | } 65 | 66 | # install current repo as package 67 | @pytest.fixture(scope="class") 68 | def packages(self): 69 | return { 70 | "packages": [ 71 | {"local": os.getcwd()} 72 | ] 73 | } 74 | 75 | 76 | # everything that goes in the "seeds" directory 77 | @pytest.fixture(scope="class") 78 | def seeds(self): 79 | return { 80 | "fact_orders_source.csv": fact_orders_source_csv 81 | } 82 | 83 | # everything that goes in the "models" directory 84 | @pytest.fixture(scope="class") 85 | def models(self): 86 | return { 87 | "fact_orders.sql": fact_orders_sql, 88 | "fact_orders.yml": fact_orders_yml, 89 | "invalid_where.sql": invalid_where_sql, 90 | "invalid_where.yml": invalid_where_yml 91 | } 92 | 93 | def test_build_completion(self,project,): 94 | # running deps to install package 95 | results = run_dbt(["deps"]) 96 | 97 | # seed seeds 98 | results = run_dbt(["seed"]) 99 | assert len(results) == 1 100 | 101 | # Here we expect the run to fail because the value provided 102 | # in the where clause isn't included in the final dataset 103 | results = run_dbt(["run"], expect_pass = False) 104 | 105 | -------------------------------------------------------------------------------- /macros/variables/get_metric_tree.sql: -------------------------------------------------------------------------------- 1 | {% macro get_metric_tree(metric_list)%} 2 | 3 | {# We are creating the metric tree here - this includes all the leafs (first level parents) 4 | , the derived metrics, and the full combination of them both #} 5 | 6 | {# This line creates the metric tree dictionary and the full_set key. 7 | Full Set contains ALL metrics that are referenced, which includes metrics in the macro 8 | AND all parent/derived metrics. #} 9 | {%- set metric_tree = {'full_set':[]} %} 10 | {# The parent set is a list of parent metrics that are NOT derived metrics. IE if 11 | metric C is built off of metric A and B, A and B would be the parent metrics because they 12 | are both upstream of Metric C AND not derived metrics themselves. #} 13 | {%- do metric_tree.update({'parent_set':[]}) -%} 14 | {# The derived set is a list of derived metrics. This includes all derived metrics referenced 15 | in the macro itself OR upstream of the metrics referenced in the macro #} 16 | {%- do metric_tree.update({'derived_set':[]}) -%} 17 | {# The base set is the list of metrics that are provided into the macro #} 18 | {%- do metric_tree.update({'base_set':[]}) -%} 19 | {# The ordered derived set is the list of derived metrics that are ordered based on their 20 | node depth. So if Metric C were downstream of Metric A and B, which were also derived metrics, 21 | Metric C would have the value of 999 (max depth) and A and B would have 998, representing that they 22 | are one depth upstream #} 23 | {%- do metric_tree.update({'ordered_derived_set':{}}) -%} 24 | 25 | {% set base_set_list = []%} 26 | {% for metric in metric_list %} 27 | {%- do base_set_list.append(metric.name) -%} 28 | {%- set metric_tree = metrics.update_metric_tree(metric ,metric_tree) -%} 29 | {% endfor %} 30 | {%- do metric_tree.update({'base_set':base_set_list}) -%} 31 | 32 | {# Now we will iterate over the metric tree and make it a unique list to account for duplicates #} 33 | {% set full_set = [] %} 34 | {% set parent_set = [] %} 35 | {% set derived_set = [] %} 36 | {% set base_set = [] %} 37 | 38 | {% for metric_name in metric_tree['full_set']|unique%} 39 | {% do full_set.append(metric_name)%} 40 | {% endfor %} 41 | {%- do metric_tree.update({'full_set':full_set}) -%} 42 | 43 | {% for metric_name in metric_tree['parent_set']|unique%} 44 | {% do parent_set.append(metric_name)%} 45 | {% endfor %} 46 | {%- do metric_tree.update({'parent_set':parent_set}) -%} 47 | 48 | {% for metric_name in metric_tree['derived_set']|unique%} 49 | {% do derived_set.append(metric_name)%} 50 | {% endfor %} 51 | {%- do metric_tree.update({'derived_set':derived_set}) -%} 52 | 53 | {% for metric in metric_tree['parent_set']|unique%} 54 | {%- do metric_tree['ordered_derived_set'].pop(metric) -%} 55 | {% endfor %} 56 | 57 | {# This section overrides the derived set by ordering the metrics on their depth so they 58 | can be correctly referenced in the downstream sql query #} 59 | {% set ordered_expression_list = []%} 60 | {% for item in metric_tree['ordered_derived_set']|dictsort(false, 'value') %} 61 | {% if item[0] in metric_tree["derived_set"]%} 62 | {% do ordered_expression_list.append(item[0])%} 63 | {% endif %} 64 | {% endfor %} 65 | {%- do metric_tree.update({'derived_set':ordered_expression_list}) -%} 66 | 67 | {%- do return(metric_tree) -%} 68 | 69 | {% endmacro %} -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/regression-report.yml: -------------------------------------------------------------------------------- 1 | name: ☣️ Regression 2 | description: Report a regression you've observed in a newer version of dbt_metrics 3 | title: "[Regression] <title>" 4 | labels: ["bug", "regression", "triage"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to fill out this regression report! 10 | - type: checkboxes 11 | attributes: 12 | label: Is this a regression in a recent version of dbt_metrics? 13 | description: > 14 | A regression is when documented functionality works as expected in an older version of dbt_metrics, 15 | and no longer works after upgrading to a newer version of dbt_metrics 16 | options: 17 | - label: I believe this is a regression in dbt_metrics functionality 18 | required: true 19 | - label: I have searched the existing issues, and I could not find an existing issue for this regression 20 | required: true 21 | - type: textarea 22 | attributes: 23 | label: Current Behavior 24 | description: A concise description of what you're experiencing. 25 | validations: 26 | required: true 27 | - type: textarea 28 | attributes: 29 | label: Expected/Previous Behavior 30 | description: A concise description of what you expected to happen. 31 | validations: 32 | required: true 33 | - type: textarea 34 | attributes: 35 | label: Steps To Reproduce 36 | description: Steps to reproduce the behavior. 37 | placeholder: | 38 | 1. In this environment... 39 | 2. With this config... 40 | 3. Run '...' 41 | 4. See error... 42 | validations: 43 | required: true 44 | - type: textarea 45 | id: logs 46 | attributes: 47 | label: Relevant log output 48 | description: | 49 | If applicable, log output to help explain your problem. 50 | render: shell 51 | validations: 52 | required: false 53 | - type: textarea 54 | attributes: 55 | label: Environment 56 | description: | 57 | examples: 58 | - **dbt-adapter & version**: dbt-snowflake v1.2.0 59 | - **dbt_metrics (working version)**: v0.3.0 60 | - **dbt_metrics (regression version)**: v0.3.1 61 | value: | 62 | - dbt-adapter & version: 63 | - dbt_metrics (working version): 64 | - dbt_metrics (regression version): 65 | render: markdown 66 | validations: 67 | required: true 68 | - type: dropdown 69 | id: database 70 | attributes: 71 | label: Which database adapter are you using with dbt? 72 | description: If the regression is specific to the database or adapter, please open the issue in that adapter's repository instead 73 | multiple: true 74 | options: 75 | - postgres 76 | - redshift 77 | - snowflake 78 | - bigquery 79 | - spark 80 | - other (mention it in "Additional Context") 81 | validations: 82 | required: false 83 | - type: textarea 84 | attributes: 85 | label: Additional Context 86 | description: | 87 | Links? References? Anything that will give us more context about the issue you are encountering! 88 | 89 | Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. 90 | validations: 91 | required: false -------------------------------------------------------------------------------- /macros/sql_gen/gen_metric_cte.sql: -------------------------------------------------------------------------------- 1 | {%- macro gen_metric_cte(metrics_dictionary, group_name, group_values, grain, dimensions, secondary_calculations, start_date, end_date, relevant_periods, calendar_dimensions) -%} 2 | {{ return(adapter.dispatch('gen_metric_cte', 'metrics')(metrics_dictionary, group_name, group_values, grain, dimensions, secondary_calculations, start_date, end_date, relevant_periods, calendar_dimensions)) }} 3 | {%- endmacro -%} 4 | 5 | {%- macro default__gen_metric_cte(metrics_dictionary, group_name, group_values, grain, dimensions, secondary_calculations, start_date, end_date, relevant_periods, calendar_dimensions) %} 6 | 7 | {%- set combined_dimensions = calendar_dimensions | list + dimensions | list -%} 8 | , {{group_name}}__final as ( 9 | {# #} 10 | select 11 | {%- if grain %} 12 | parent_metric_cte.date_{{grain}}, 13 | {%- if secondary_calculations | length > 0 -%} 14 | {% for period in relevant_periods %} 15 | parent_metric_cte.date_{{ period }}, 16 | {%- endfor -%} 17 | {%- endif -%} 18 | {%- endif -%} 19 | 20 | {%- for calendar_dim in calendar_dimensions %} 21 | parent_metric_cte.{{ calendar_dim }}, 22 | {%- endfor %} 23 | 24 | {%- for dim in dimensions %} 25 | parent_metric_cte.{{ dim }}, 26 | {%- endfor %} 27 | 28 | {%- for metric_name in group_values.metric_names -%} 29 | {# TODO: coalesce based on the value. Need to bring this config #} 30 | {%- if not metrics_dictionary[metric_name].get("config").get("treat_null_values_as_zero", True) %} 31 | {{ metric_name }} 32 | {%- else %} 33 | coalesce({{ metric_name }}, 0) as {{ metric_name }} 34 | {%- endif %} 35 | {%- if not loop.last-%},{%endif%} 36 | {%- endfor %} 37 | 38 | {%- if secondary_calculations | length > 0 %} 39 | from {{group_name}}__spine_time as parent_metric_cte 40 | left outer join {{group_name}}__aggregate 41 | using (date_{{grain}} {%- if combined_dimensions | length > 0 -%}, {{ combined_dimensions | join(", ") }} {%-endif-%} ) 42 | 43 | {% if not start_date or not end_date -%} 44 | where ( 45 | {% if not start_date and not end_date -%} 46 | parent_metric_cte.date_{{grain}} >= ( 47 | select 48 | min(case when has_data then date_{{grain}} end) 49 | from {{group_name}}__aggregate 50 | ) 51 | and parent_metric_cte.date_{{grain}} <= ( 52 | select 53 | max(case when has_data then date_{{grain}} end) 54 | from {{group_name}}__aggregate 55 | ) 56 | {% elif not start_date and end_date -%} 57 | parent_metric_cte.date_{{grain}} >= ( 58 | select 59 | min(case when has_data then date_{{grain}} end) 60 | from {{group_name}}__aggregate 61 | ) 62 | {% elif start_date and not end_date -%} 63 | parent_metric_cte.date_{{grain}} <= ( 64 | select 65 | max(case when has_data then date_{{grain}} end) 66 | from {{group_name}}__aggregate 67 | ) 68 | {%- endif %} 69 | ) 70 | {%- endif %} 71 | 72 | {%- else %} 73 | from {{group_name}}__aggregate as parent_metric_cte 74 | {%- endif %} 75 | ) 76 | 77 | {% endmacro %} 78 | -------------------------------------------------------------------------------- /integration_tests/models/custom_calendar.sql: -------------------------------------------------------------------------------- 1 | with days as ( 2 | 3 | 4 | with rawdata as ( 5 | 6 | 7 | with p as ( 8 | select 0 as generated_number union all select 1 9 | ), unioned as ( 10 | 11 | select 12 | 13 | 14 | p0.generated_number * power(2, 0) 15 | + 16 | 17 | p1.generated_number * power(2, 1) 18 | + 19 | 20 | p2.generated_number * power(2, 2) 21 | + 22 | 23 | p3.generated_number * power(2, 3) 24 | + 25 | 26 | p4.generated_number * power(2, 4) 27 | + 28 | 29 | p5.generated_number * power(2, 5) 30 | + 31 | 32 | p6.generated_number * power(2, 6) 33 | + 34 | 35 | p7.generated_number * power(2, 7) 36 | + 37 | 38 | p8.generated_number * power(2, 8) 39 | + 40 | 41 | p9.generated_number * power(2, 9) 42 | + 43 | 44 | p10.generated_number * power(2, 10) 45 | + 46 | 47 | p11.generated_number * power(2, 11) 48 | + 49 | 50 | p12.generated_number * power(2, 12) 51 | + 52 | 53 | p13.generated_number * power(2, 13) 54 | 55 | 56 | + 1 57 | as generated_number 58 | 59 | from 60 | 61 | 62 | p as p0 63 | cross join 64 | 65 | p as p1 66 | cross join 67 | 68 | p as p2 69 | cross join 70 | 71 | p as p3 72 | cross join 73 | 74 | p as p4 75 | cross join 76 | 77 | p as p5 78 | cross join 79 | 80 | p as p6 81 | cross join 82 | 83 | p as p7 84 | cross join 85 | 86 | p as p8 87 | cross join 88 | 89 | p as p9 90 | cross join 91 | 92 | p as p10 93 | cross join 94 | 95 | p as p11 96 | cross join 97 | 98 | p as p12 99 | cross join 100 | 101 | p as p13 102 | 103 | ) 104 | 105 | select * 106 | from unioned 107 | where generated_number <= 14610 108 | order by generated_number 109 | 110 | ), 111 | 112 | all_periods as ( 113 | 114 | select ( 115 | 116 | 117 | dateadd( 118 | day, 119 | row_number() over (order by 1) - 1, 120 | cast('1990-01-01' as date) 121 | ) 122 | 123 | 124 | ) as date_day 125 | from rawdata 126 | 127 | ), 128 | 129 | filtered as ( 130 | 131 | select * 132 | from all_periods 133 | where date_day <= cast('2030-01-01' as date) 134 | 135 | ) 136 | 137 | select * from filtered 138 | 139 | 140 | ), 141 | 142 | final as ( 143 | select 144 | cast(date_day as date) as date_day, 145 | {% if target.type == 'bigquery' %} 146 | --BQ starts its weeks on Sunday. I don't actually care which day it runs on for auto testing purposes, just want it to be consistent with the other seeds 147 | cast({{ date_trunc('week(MONDAY)', 'date_day') }} as date) as date_week, 148 | {% else %} 149 | cast({{ date_trunc('week', 'date_day') }} as date) as date_week, 150 | {% endif %} 151 | cast({{ date_trunc('month', 'date_day') }} as date) as date_month, 152 | cast({{ date_trunc('quarter', 'date_day') }} as date) as date_quarter, 153 | '2022-01-01' as date_test, 154 | cast({{ date_trunc('year', 'date_day') }} as date) as date_year, 155 | true as is_weekend 156 | from days 157 | ) 158 | 159 | select * from final 160 | -------------------------------------------------------------------------------- /macros/calculate.sql: -------------------------------------------------------------------------------- 1 | {% macro calculate(metric_list, grain=none, dimensions=[], secondary_calculations=[], start_date=none, end_date=none, where=none, date_alias=none) %} 2 | {{ return(adapter.dispatch('calculate', 'metrics')(metric_list, grain, dimensions, secondary_calculations, start_date, end_date, where, date_alias)) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__calculate(metric_list, grain=none, dimensions=[], secondary_calculations=[], start_date=none, end_date=none, where=none, date_alias=none) %} 7 | {#- Need this here, since the actual ref is nested within loops/conditions: -#} 8 | -- depends on: {{ ref(var('dbt_metrics_calendar_model', 'dbt_metrics_default_calendar')) }} 9 | 10 | {#- ############ 11 | VARIABLE SETTING - Creating the metric tree and making sure metric list is a list! 12 | ############ -#} 13 | 14 | {%- if execute %} 15 | {% do exceptions.warn( 16 | "WARNING: dbt_metrics is going to be deprecated in dbt-core 1.6 in \ 17 | July 2023 as part of the migration to MetricFlow. This package will \ 18 | continue to work with dbt-core 1.5 but a 1.6 version will not be \ 19 | released. If you have any questions, please join us in the #dbt-core-metrics in the dbt Community Slack") %} 20 | {%- endif %} 21 | 22 | {%- if metric_list is not iterable -%} 23 | {%- set metric_list = [metric_list] -%} 24 | {%- endif -%} 25 | 26 | {%- set metric_tree = metrics.get_metric_tree(metric_list=metric_list) -%} 27 | 28 | {#- Here we are creating the metrics dictionary which contains all of the metric information needed for sql gen. -#} 29 | {%- set metrics_dictionary = metrics.get_metrics_dictionary(metric_tree=metric_tree) -%} 30 | 31 | {#- ############ 32 | VALIDATION - Make sure everything is good! 33 | ############ -#} 34 | 35 | {%- if not execute -%} 36 | {%- do return("Did not execute") -%} 37 | {%- endif -%} 38 | 39 | {%- if not metric_list -%} 40 | {%- do exceptions.raise_compiler_error("No metric or metrics provided") -%} 41 | {%- endif -%} 42 | 43 | {%- do metrics.validate_timestamp(grain=grain, metric_tree=metric_tree, metrics_dictionary=metrics_dictionary, dimensions=dimensions) -%} 44 | 45 | {%- do metrics.validate_grain(grain=grain, metric_tree=metric_tree, metrics_dictionary=metrics_dictionary, secondary_calculations=secondary_calculations) -%} 46 | 47 | {%- do metrics.validate_derived_metrics(metric_tree=metric_tree) -%} 48 | 49 | {%- do metrics.validate_dimension_list(dimensions=dimensions, metric_tree=metric_tree, metrics_dictionary=metrics_dictionary) -%} 50 | 51 | {# {%- do metrics.validate_metric_config(metrics_dictionary=metrics_dictionary) -%} #} 52 | 53 | {%- do metrics.validate_where(where=where) -%} 54 | 55 | {%- do metrics.validate_secondary_calculations(metric_tree=metric_tree, metrics_dictionary=metrics_dictionary, grain=grain, secondary_calculations=secondary_calculations) -%} 56 | 57 | {%- do metrics.validate_calendar_model() -%} 58 | 59 | {#- ############ 60 | SQL GENERATION - Lets build that SQL! 61 | ############ -#} 62 | 63 | {%- set sql = metrics.get_metric_sql( 64 | metrics_dictionary=metrics_dictionary, 65 | grain=grain, 66 | dimensions=dimensions, 67 | secondary_calculations=secondary_calculations, 68 | start_date=start_date, 69 | end_date=end_date, 70 | where=where, 71 | date_alias=date_alias, 72 | metric_tree=metric_tree 73 | ) %} 74 | 75 | ({{ sql }}) metric_subq 76 | 77 | {%- endmacro -%} 78 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.yml: -------------------------------------------------------------------------------- 1 | name: 🐞 Bug 2 | description: Report a bug or an issue you've found with dbt_metrics 3 | title: "[Bug] <title>" 4 | labels: ["bug", "triage"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to fill out this bug report! 10 | - type: checkboxes 11 | attributes: 12 | label: Is this a new bug in dbt_metrics? 13 | description: > 14 | In other words, is this an error, flaw, failure or fault in the package? 15 | 16 | If this is a bug that broke existing functionality that used to work, please open a regression issue. 17 | If this is a bug experienced while using dbt Cloud, please report to [support](mailto:support@getdbt.com). 18 | If this is a request for help or troubleshooting code in your own dbt project, please join our [dbt Community Slack](https://www.getdbt.com/community/join-the-community/) or open a [Discussion question](https://github.com/dbt-labs/docs.getdbt.com/discussions). 19 | 20 | Please search to see if an issue already exists for the bug you encountered. 21 | options: 22 | - label: I believe this is a new bug in dbt_metrics 23 | required: true 24 | - label: I have searched the existing issues, and I could not find an existing issue for this bug 25 | required: true 26 | - type: textarea 27 | attributes: 28 | label: Current Behavior 29 | description: A concise description of what you're experiencing. 30 | validations: 31 | required: true 32 | - type: textarea 33 | attributes: 34 | label: Expected Behavior 35 | description: A concise description of what you expected to happen. 36 | validations: 37 | required: true 38 | - type: textarea 39 | attributes: 40 | label: Steps To Reproduce 41 | description: Steps to reproduce the behavior. 42 | placeholder: | 43 | 1. In this environment... 44 | 2. With this config... 45 | 3. Run '...' 46 | 4. See error... 47 | validations: 48 | required: true 49 | - type: textarea 50 | id: logs 51 | attributes: 52 | label: Relevant log output 53 | description: | 54 | If applicable, log output to help explain your problem. 55 | render: shell 56 | validations: 57 | required: false 58 | - type: textarea 59 | attributes: 60 | label: Environment 61 | description: | 62 | examples: 63 | - **dbt-adapter & version**: dbt-snowflake v1.2.0 64 | - **dbt_metrics version**: v0.3.0 65 | value: | 66 | - dbt-adapter & version: 67 | - dbt_metrics version: 68 | render: markdown 69 | validations: 70 | required: false 71 | - type: dropdown 72 | id: database 73 | attributes: 74 | label: Which database adapter are you using with dbt? 75 | description: If the bug is specific to the database or adapter, please open the issue in that adapter's repository instead 76 | multiple: true 77 | options: 78 | - postgres 79 | - redshift 80 | - snowflake 81 | - bigquery 82 | - databricks 83 | - spark 84 | - other (mention it in "Additional Context") 85 | validations: 86 | required: false 87 | - type: textarea 88 | attributes: 89 | label: Additional Context 90 | description: | 91 | Links? References? Anything that will give us more context about the issue you are encountering! 92 | 93 | Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. 94 | validations: 95 | required: false 96 | --------------------------------------------------------------------------------